Cardiovascular Disease Analysis

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn
from itertools import product

# data transformation & splitting
from sklearn.preprocessing import RobustScaler, label_binarize
from sklearn.model_selection import train_test_split, GridSearchCV

# decision tree, logistic
from sklearn.tree import DecisionTreeClassifier, export_graphviz
import graphviz
from sklearn.linear_model import LogisticRegressionCV

# clustering analysis
from sklearn.cluster import KMeans, AgglomerativeClustering, DBSCAN
import scipy.cluster.hierarchy as sch
from sklearn.mixture import GaussianMixture
from sklearn.metrics import jaccard_score, adjusted_rand_score, silhouette_score, calinski_harabasz_score, roc_curve, auc, accuracy_score, classification_report, confusion_matrix
from sklearn.metrics.cluster import contingency_matrix

# ensemble methods, MLP
from sklearn.ensemble import BaggingClassifier, RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier
from sklearn.svm import LinearSVC, SVC
from sklearn.neural_network import MLPClassifier
#------------------------------------------------------------------------------------------   
# Input data files are available in the read-only "../input/" directory
import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# read data
df = pd.read_csv('cardio_train.csv', sep = ';', index_col = 'id')

# preview
pd.options.display.float_format = '{:,.2f}'.format
display(df.head())

	age	gender	height	weight	ap_hi	ap_lo	cholesterol	gluc	smoke	alco	active	cardio
id
0	18393	2	168	62.00	110	80	1	1	0	0	1	0
1	20228	1	156	85.00	140	90	3	1	0	0	1	1
2	18857	1	165	64.00	130	70	3	1	0	0	0	1
3	17623	2	169	82.00	150	100	1	1	0	0	1	1
4	17474	1	156	56.00	100	60	1	1	0	0	0	0

Data Cleaning

Check missing values:

# check blank rows
df.isnull().sum() 

age            0
gender         0
height         0
weight         0
ap_hi          0
ap_lo          0
cholesterol    0
gluc           0
smoke          0
alco           0
active         0
cardio         0
dtype: int64

None of the variables have missing values.

Examining the variables:

# before data cleaning
display(df.describe())

	age	gender	height	weight	ap_hi	ap_lo	cholesterol	gluc	smoke	alco	active	cardio
count	70,000.00	70,000.00	70,000.00	70,000.00	70,000.00	70,000.00	70,000.00	70,000.00	70,000.00	70,000.00	70,000.00	70,000.00
mean	19,468.87	1.35	164.36	74.21	128.82	96.63	1.37	1.23	0.09	0.05	0.80	0.50
std	2,467.25	0.48	8.21	14.40	154.01	188.47	0.68	0.57	0.28	0.23	0.40	0.50
min	10,798.00	1.00	55.00	10.00	-150.00	-70.00	1.00	1.00	0.00	0.00	0.00	0.00
25%	17,664.00	1.00	159.00	65.00	120.00	80.00	1.00	1.00	0.00	0.00	1.00	0.00
50%	19,703.00	1.00	165.00	72.00	120.00	80.00	1.00	1.00	0.00	0.00	1.00	0.00
75%	21,327.00	2.00	170.00	82.00	140.00	90.00	2.00	1.00	0.00	0.00	1.00	1.00
max	23,713.00	2.00	250.00	200.00	16,020.00	11,000.00	3.00	3.00	1.00	1.00	1.00	1.00

Age : Converting to years for ease of understanding.

# convert age from days to years
df['age'] = df['age']/365

Gender: Converting female to 0 and male to 1

# convert gender to 0=female and 1=male
df['gender'] = df['gender'] - 1

Height and Weight : Using BMI as an indicator to remove records that do not make sense.

# calculate BMI
df['BMI'] = df['weight']/(np.power(df['height']/100, 2))

# BMI Distribution
print(df['BMI'].describe())
seaborn.histplot(data = df, x = 'BMI', bins = 100)
plt.show()

count   70,000.00
mean        27.56
std          6.09
min          3.47
25%         23.88
50%         26.37
75%         30.22
max        298.67
Name: BMI, dtype: float64

# remove BMI > 150
drop_criteria_bmi = df[df['BMI'] > 150].index

# number of records to be removed
print(drop_criteria_bmi.size)

# remove records
df.drop(drop_criteria_bmi, inplace = True)

13

AP_HI and AP_LO : Blood pressure should always be positive, not exceeding a certain threshold (300). AP_HI > AP_LO checking should be enforced.

# ap_hi is higher than 250 or lower than 60
drop_criteria_aphi = df[(df['ap_hi'] > 210) | (df['ap_hi'] < 60)].index

# ap_lo is higher than 200 or lower than 10
drop_criteria_aplo = df[(df['ap_lo'] > 140) | (df['ap_lo'] < 30)].index

# ap_lo is higher than 'ap_hi
drop_criteria_ap = df[df['ap_lo'] > df['ap_hi']].index 

# number of records to be removed
drop_criteria = drop_criteria_aphi.union(drop_criteria_aplo)
drop_criteria.union(drop_criteria_ap)
print(drop_criteria.size)

1276

# remove the records
df.drop(drop_criteria, inplace = True)

Data cleaning result:

# after data cleaning
display(df.describe())

# distribution of response variable
display(pd.DataFrame(df['cardio'].value_counts()))

	age	gender	height	weight	ap_hi	ap_lo	cholesterol	gluc	smoke	alco	active	cardio	BMI
count	68,711.00	68,711.00	68,711.00	68,711.00	68,711.00	68,711.00	68,711.00	68,711.00	68,711.00	68,711.00	68,711.00	68,711.00	68,711.00
mean	53.33	0.35	164.38	74.11	126.57	81.33	1.36	1.23	0.09	0.05	0.80	0.49	27.49
std	6.76	0.48	8.08	14.30	16.62	9.51	0.68	0.57	0.28	0.23	0.40	0.50	5.51
min	29.58	0.00	64.00	11.00	60.00	30.00	1.00	1.00	0.00	0.00	0.00	0.00	3.47
25%	48.37	0.00	159.00	65.00	120.00	80.00	1.00	1.00	0.00	0.00	1.00	0.00	23.88
50%	53.98	0.00	165.00	72.00	120.00	80.00	1.00	1.00	0.00	0.00	1.00	0.00	26.35
75%	58.42	1.00	170.00	82.00	140.00	90.00	1.00	1.00	0.00	0.00	1.00	1.00	30.12
max	64.97	1.00	250.00	200.00	210.00	140.00	3.00	3.00	1.00	1.00	1.00	1.00	148.93

	count
cardio
0	34725
1	33986

The data is balanced. There is a fairly even split between individuals with the disease and without the disease.

Visualizing Variables

def pie_chart(df, col, labels):
    data = df[col].value_counts().to_numpy()
    def absolute_value(val):
        a  = np.round(val / 100 * data.sum(), 0)
        return str('%0.0f' % a) + '\n(' + ('%0.2f' % val) + '%)'
    plt.pie(data, labels = labels, autopct=absolute_value)
    plt.legend(title=col)
    plt.show() 

pie_chart(df, 'gender', ['Female', 'Male'])
pie_chart(df, 'cardio', ['No', 'Yes'])
pie_chart(df, 'cholesterol', ['Normal', 'Above normal', 'Well above normal'])
pie_chart(df, 'gluc', ['Normal', 'Above normal', 'Well above normal'])
pie_chart(df, 'smoke', ['No', 'Yes'])
pie_chart(df, 'alco', ['No', 'Yes'])
pie_chart(df, 'active', ['No', 'Yes'])

df_subset = df[['age', 'height', 'weight', 'ap_hi', 'ap_lo']]
flierprops = dict(markerfacecolor='lightblue', marker='o',markeredgecolor='lightblue') 
# <=> rs = {'markerfacecolor'='lightblue', 'marker'='o'}
boxprops = dict(facecolor='lightblue',color = 'lightblue') # color: box line color; facecolor: fill-in color
plt.figure(figsize=(10, 5))
plt.boxplot(df_subset.values,labels=df_subset.columns,
           flierprops=flierprops,boxprops=boxprops,
            patch_artist=True)
plt.show()
df_subset.boxplot()

<Axes: >

from sklearn.preprocessing import QuantileTransformer
quantile_transformer = QuantileTransformer(random_state=0)
X_trans = quantile_transformer.fit_transform(df_subset)
pd.DataFrame(X_trans, columns=df_subset.columns).hist(bins = 5)
plt.tight_layout()

flierprops = dict(markerfacecolor='lightblue', marker='o',markeredgecolor='lightblue') 
# <=> rs = {'markerfacecolor'='lightblue', 'marker'='o'}
boxprops = dict(facecolor='lightblue',color = 'lightblue') # color: box line color; facecolor: fill-in color
plt.figure(figsize=(10, 5))
plt.boxplot(df_subset.values,labels=df_subset.columns,
           flierprops=flierprops,boxprops=boxprops,
            patch_artist=True)
plt.show()

%%time

# pair-plot
seaborn.pairplot(df, vars = ['age', 'height', 'weight', 'ap_hi', 'ap_lo'], hue = 'cardio')
plt.show()

CPU times: user 35.9 s, sys: 666 ms, total: 36.6 s
Wall time: 31.7 s

Pairplots by gender

# gender pair-plot
seaborn.pairplot(df[df.gender == 0], vars = ['age', 'height', 'weight', 'ap_hi', 'ap_lo'], hue = 'cardio')
plt.show()
seaborn.pairplot(df[df.gender == 1], vars = ['age', 'height', 'weight', 'ap_hi', 'ap_lo'], hue = 'cardio')
plt.show()

Correlation heatmap

# correlation heatmap
plt.figure(figsize=(16, 8))
seaborn.heatmap(df.corr(), annot=True, fmt='.3f')

<Axes: >

Data Transformation & Train-Test Splitting

# explanatory variables
x = df.drop(columns = ['cardio', 'BMI'])

# response variable
y = df['cardio']

# split data
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.4, random_state = 8017)

# RobustScaler ALL variables
scaler = RobustScaler()
scaler.fit(x_train)
x_train = pd.DataFrame(scaler.transform(x_train), index=x_train.index, columns=x_train.columns)
x_test = pd.DataFrame(scaler.transform(x_test), index=x_test.index, columns=x_test.columns)

pd.options.display.float_format = '{:,.4f}'.format
display(x_train.head(5))
display(x_train.describe())

	age	gender	height	weight	ap_hi	ap_lo	cholesterol	gluc	smoke	alco	active
id
2319	-0.0293	0.0000	-0.7273	0.0000	0.0000	0.0000	1.0000	1.0000	0.0000	0.0000	0.0000
47870	0.4019	0.0000	-0.5455	0.9412	-0.5000	-1.0000	0.0000	0.0000	0.0000	0.0000	0.0000
65607	-0.4431	1.0000	0.8182	0.4706	1.0000	0.0000	1.0000	0.0000	1.0000	0.0000	0.0000
58447	0.7868	0.0000	0.3636	1.9412	0.0000	0.0000	2.0000	0.0000	0.0000	0.0000	0.0000
60084	0.8070	0.0000	-0.4545	-0.6471	0.0000	0.0000	0.0000	0.0000	0.0000	0.0000	0.0000

	age	gender	height	weight	ap_hi	ap_lo	cholesterol	gluc	smoke	alco	active
count	41,226.0000	41,226.0000	41,226.0000	41,226.0000	41,226.0000	41,226.0000	41,226.0000	41,226.0000	41,226.0000	41,226.0000	41,226.0000
mean	-0.0664	0.3509	-0.0540	0.1218	0.3254	0.1269	0.3652	0.2237	0.0891	0.0540	-0.1961
std	0.6746	0.4773	0.7340	0.8407	0.8318	0.9533	0.6792	0.5699	0.2848	0.2261	0.3970
min	-2.4300	0.0000	-9.1818	-3.5882	-2.5000	-5.0000	0.0000	0.0000	0.0000	0.0000	-1.0000
25%	-0.5593	0.0000	-0.5455	-0.4118	0.0000	0.0000	0.0000	0.0000	0.0000	0.0000	0.0000
50%	0.0000	0.0000	0.0000	0.0000	0.0000	0.0000	0.0000	0.0000	0.0000	0.0000	0.0000
75%	0.4407	1.0000	0.4545	0.5882	1.0000	1.0000	1.0000	0.0000	0.0000	0.0000	0.0000
max	1.0929	1.0000	7.7273	7.5294	4.5000	6.0000	2.0000	2.0000	1.0000	1.0000	0.0000

Decision Tree

%%time

# parameters candidates
parameters = {'max_depth':range(2,32)}

# fitting
DecisionTree_GSCV = GridSearchCV(DecisionTreeClassifier(random_state=8017), 
                                 parameters, n_jobs=-1, verbose=3, return_train_score=True)
DecisionTree_GSCV.fit(x_train, y_train)
DecisionTree_model = DecisionTree_GSCV.best_estimator_

Fitting 5 folds for each of 30 candidates, totalling 150 fits

[CV 4/5] END ...max_depth=2;, score=(train=0.715, test=0.706) total time=   0.0s
[CV 2/5] END ...max_depth=2;, score=(train=0.714, test=0.714) total time=   0.0s
[CV 3/5] END ...max_depth=2;, score=(train=0.713, test=0.716) total time=   0.0s
[CV 1/5] END ...max_depth=3;, score=(train=0.724, test=0.730) total time=   0.0s
[CV 1/5] END ...max_depth=2;, score=(train=0.713, test=0.716) total time=   0.0s
[CV 3/5] END ...max_depth=3;, score=(train=0.724, test=0.729) total time=   0.0s
[CV 5/5] END ...max_depth=2;, score=(train=0.713, test=0.715) total time=   0.0s
[CV 5/5] END ...max_depth=3;, score=(train=0.725, test=0.726) total time=   0.0s
[CV 4/5] END ...max_depth=3;, score=(train=0.727, test=0.717) total time=   0.0s
[CV 2/5] END ...max_depth=3;, score=(train=0.725, test=0.726) total time=   0.0s
[CV 1/5] END ...max_depth=4;, score=(train=0.728, test=0.732) total time=   0.0s
[CV 2/5] END ...max_depth=4;, score=(train=0.729, test=0.727) total time=   0.0s
[CV 3/5] END ...max_depth=4;, score=(train=0.727, test=0.735) total time=   0.0s
[CV 5/5] END ...max_depth=4;, score=(train=0.729, test=0.729) total time=   0.0s
[CV 4/5] END ...max_depth=4;, score=(train=0.731, test=0.719) total time=   0.0s
[CV 2/5] END ...max_depth=5;, score=(train=0.734, test=0.727) total time=   0.1s
[CV 1/5] END ...max_depth=5;, score=(train=0.733, test=0.738) total time=   0.1s
[CV 3/5] END ...max_depth=5;, score=(train=0.732, test=0.738) total time=   0.1s
[CV 5/5] END ...max_depth=5;, score=(train=0.735, test=0.730) total time=   0.1s
[CV 4/5] END ...max_depth=5;, score=(train=0.736, test=0.724) total time=   0.1s
[CV 1/5] END ...max_depth=6;, score=(train=0.734, test=0.736) total time=   0.1s
[CV 2/5] END ...max_depth=6;, score=(train=0.736, test=0.730) total time=   0.0s
[CV 5/5] END ...max_depth=6;, score=(train=0.736, test=0.730) total time=   0.1s
[CV 4/5] END ...max_depth=6;, score=(train=0.737, test=0.724) total time=   0.1s
[CV 3/5] END ...max_depth=6;, score=(train=0.734, test=0.737) total time=   0.1s
[CV 2/5] END ...max_depth=7;, score=(train=0.739, test=0.726) total time=   0.1s
[CV 3/5] END ...max_depth=7;, score=(train=0.738, test=0.737) total time=   0.1s
[CV 4/5] END ...max_depth=7;, score=(train=0.741, test=0.722) total time=   0.1s
[CV 1/5] END ...max_depth=7;, score=(train=0.737, test=0.736) total time=   0.1s
[CV 5/5] END ...max_depth=7;, score=(train=0.739, test=0.727) total time=   0.1s
[CV 1/5] END ...max_depth=8;, score=(train=0.742, test=0.735) total time=   0.1s
[CV 2/5] END ...max_depth=8;, score=(train=0.744, test=0.727) total time=   0.1s
[CV 3/5] END ...max_depth=8;, score=(train=0.743, test=0.735) total time=   0.1s
[CV 5/5] END ...max_depth=8;, score=(train=0.744, test=0.725) total time=   0.1s
[CV 4/5] END ...max_depth=8;, score=(train=0.744, test=0.723) total time=   0.1s
[CV 2/5] END ...max_depth=9;, score=(train=0.750, test=0.726) total time=   0.1s
[CV 1/5] END ...max_depth=9;, score=(train=0.748, test=0.732) total time=   0.1s
[CV 3/5] END ...max_depth=9;, score=(train=0.749, test=0.728) total time=   0.1s
[CV 4/5] END ...max_depth=9;, score=(train=0.752, test=0.717) total time=   0.1s
[CV 5/5] END ...max_depth=9;, score=(train=0.750, test=0.726) total time=   0.1s
[CV 1/5] END ..max_depth=10;, score=(train=0.759, test=0.732) total time=   0.1s
[CV 2/5] END ..max_depth=10;, score=(train=0.759, test=0.721) total time=   0.1s
[CV 3/5] END ..max_depth=10;, score=(train=0.758, test=0.727) total time=   0.1s
[CV 4/5] END ..max_depth=10;, score=(train=0.759, test=0.714) total time=   0.1s
[CV 5/5] END ..max_depth=10;, score=(train=0.758, test=0.724) total time=   0.1s
[CV 1/5] END ..max_depth=11;, score=(train=0.768, test=0.728) total time=   0.1s
[CV 2/5] END ..max_depth=11;, score=(train=0.768, test=0.716) total time=   0.1s
[CV 3/5] END ..max_depth=11;, score=(train=0.768, test=0.726) total time=   0.1s
[CV 4/5] END ..max_depth=11;, score=(train=0.768, test=0.711) total time=   0.1s
[CV 5/5] END ..max_depth=11;, score=(train=0.768, test=0.719) total time=   0.1s
[CV 2/5] END ..max_depth=12;, score=(train=0.782, test=0.707) total time=   0.1s
[CV 1/5] END ..max_depth=12;, score=(train=0.778, test=0.727) total time=   0.1s
[CV 3/5] END ..max_depth=12;, score=(train=0.777, test=0.719) total time=   0.1s
[CV 4/5] END ..max_depth=12;, score=(train=0.780, test=0.710) total time=   0.1s
[CV 5/5] END ..max_depth=12;, score=(train=0.780, test=0.716) total time=   0.1s
[CV 1/5] END ..max_depth=13;, score=(train=0.789, test=0.720) total time=   0.1s
[CV 2/5] END ..max_depth=13;, score=(train=0.796, test=0.706) total time=   0.1s
[CV 4/5] END ..max_depth=13;, score=(train=0.794, test=0.707) total time=   0.1s
[CV 1/5] END ..max_depth=14;, score=(train=0.802, test=0.713) total time=   0.1s
[CV 3/5] END ..max_depth=14;, score=(train=0.803, test=0.711) total time=   0.1s
[CV 5/5] END ..max_depth=14;, score=(train=0.803, test=0.707) total time=   0.1s
[CV 2/5] END ..max_depth=15;, score=(train=0.826, test=0.694) total time=   0.1s
[CV 3/5] END ..max_depth=13;, score=(train=0.790, test=0.716) total time=   0.1s
[CV 4/5] END ..max_depth=15;, score=(train=0.825, test=0.689) total time=   0.1s
[CV 5/5] END ..max_depth=13;, score=(train=0.791, test=0.712) total time=   0.1s
[CV 1/5] END ..max_depth=16;, score=(train=0.831, test=0.699) total time=   0.1s
[CV 4/5] END ..max_depth=14;, score=(train=0.808, test=0.698) total time=   0.1s
[CV 2/5] END ..max_depth=14;, score=(train=0.810, test=0.704) total time=   0.1s
[CV 1/5] END ..max_depth=15;, score=(train=0.816, test=0.708) total time=   0.1s
[CV 3/5] END ..max_depth=15;, score=(train=0.818, test=0.705) total time=   0.1s
[CV 5/5] END ..max_depth=15;, score=(train=0.817, test=0.702) total time=   0.1s
[CV 3/5] END ..max_depth=16;, score=(train=0.833, test=0.703) total time=   0.1s
[CV 5/5] END ..max_depth=16;, score=(train=0.830, test=0.698) total time=   0.1s
[CV 2/5] END ..max_depth=16;, score=(train=0.841, test=0.692) total time=   0.1s
[CV 2/5] END ..max_depth=17;, score=(train=0.858, test=0.686) total time=   0.1s
[CV 4/5] END ..max_depth=17;, score=(train=0.856, test=0.682) total time=   0.1s
[CV 1/5] END ..max_depth=18;, score=(train=0.863, test=0.689) total time=   0.1s
[CV 3/5] END ..max_depth=18;, score=(train=0.863, test=0.679) total time=   0.1s
[CV 1/5] END ..max_depth=17;, score=(train=0.847, test=0.697) total time=   0.1s
[CV 4/5] END ..max_depth=16;, score=(train=0.840, test=0.688) total time=   0.1s
[CV 5/5] END ..max_depth=18;, score=(train=0.859, test=0.678) total time=   0.1s
[CV 2/5] END ..max_depth=19;, score=(train=0.888, test=0.682) total time=   0.1s
[CV 3/5] END ..max_depth=17;, score=(train=0.847, test=0.696) total time=   0.1s
[CV 5/5] END ..max_depth=17;, score=(train=0.846, test=0.684) total time=   0.1s
[CV 2/5] END ..max_depth=18;, score=(train=0.873, test=0.683) total time=   0.1s
[CV 4/5] END ..max_depth=18;, score=(train=0.870, test=0.677) total time=   0.1s
[CV 4/5] END ..max_depth=19;, score=(train=0.884, test=0.679) total time=   0.1s
[CV 1/5] END ..max_depth=20;, score=(train=0.892, test=0.679) total time=   0.1s
[CV 3/5] END ..max_depth=19;, score=(train=0.879, test=0.671) total time=   0.1s
[CV 1/5] END ..max_depth=19;, score=(train=0.877, test=0.684) total time=   0.1s
[CV 3/5] END ..max_depth=20;, score=(train=0.894, test=0.669) total time=   0.1s
[CV 5/5] END ..max_depth=20;, score=(train=0.890, test=0.665) total time=   0.1s
[CV 2/5] END ..max_depth=21;, score=(train=0.914, test=0.671) total time=   0.1s
[CV 4/5] END ..max_depth=21;, score=(train=0.911, test=0.670) total time=   0.1s
[CV 2/5] END ..max_depth=20;, score=(train=0.901, test=0.674) total time=   0.1s
[CV 5/5] END ..max_depth=19;, score=(train=0.876, test=0.671) total time=   0.1s
[CV 1/5] END ..max_depth=22;, score=(train=0.918, test=0.672) total time=   0.1s
[CV 3/5] END ..max_depth=22;, score=(train=0.921, test=0.656) total time=   0.1s
[CV 4/5] END ..max_depth=20;, score=(train=0.897, test=0.670) total time=   0.1s
[CV 1/5] END ..max_depth=21;, score=(train=0.906, test=0.676) total time=   0.1s
[CV 5/5] END ..max_depth=21;, score=(train=0.904, test=0.660) total time=   0.1s
[CV 2/5] END ..max_depth=23;, score=(train=0.935, test=0.660) total time=   0.1s
[CV 3/5] END ..max_depth=21;, score=(train=0.908, test=0.665) total time=   0.2s
[CV 5/5] END ..max_depth=22;, score=(train=0.917, test=0.655) total time=   0.1s
[CV 2/5] END ..max_depth=22;, score=(train=0.924, test=0.666) total time=   0.1s
[CV 4/5] END ..max_depth=23;, score=(train=0.933, test=0.658) total time=   0.1s
[CV 4/5] END ..max_depth=22;, score=(train=0.922, test=0.664) total time=   0.1s
[CV 1/5] END ..max_depth=24;, score=(train=0.939, test=0.667) total time=   0.1s
[CV 3/5] END ..max_depth=23;, score=(train=0.933, test=0.657) total time=   0.1s
[CV 1/5] END ..max_depth=23;, score=(train=0.930, test=0.669) total time=   0.1s
[CV 3/5] END ..max_depth=24;, score=(train=0.945, test=0.652) total time=   0.1s
[CV 5/5] END ..max_depth=23;, score=(train=0.929, test=0.649) total time=   0.1s
[CV 2/5] END ..max_depth=25;, score=(train=0.954, test=0.652) total time=   0.1s
[CV 5/5] END ..max_depth=24;, score=(train=0.940, test=0.650) total time=   0.2s
[CV 2/5] END ..max_depth=24;, score=(train=0.946, test=0.656) total time=   0.1s
[CV 4/5] END ..max_depth=25;, score=(train=0.951, test=0.653) total time=   0.1s
[CV 1/5] END ..max_depth=26;, score=(train=0.956, test=0.658) total time=   0.1s
[CV 4/5] END ..max_depth=24;, score=(train=0.941, test=0.657) total time=   0.1s
[CV 3/5] END ..max_depth=26;, score=(train=0.962, test=0.650) total time=   0.1s
[CV 3/5] END ..max_depth=25;, score=(train=0.954, test=0.650) total time=   0.1s
[CV 1/5] END ..max_depth=25;, score=(train=0.948, test=0.662) total time=   0.1s
[CV 5/5] END ..max_depth=26;, score=(train=0.959, test=0.641) total time=   0.1s
[CV 5/5] END ..max_depth=25;, score=(train=0.950, test=0.643) total time=   0.1s
[CV 2/5] END ..max_depth=27;, score=(train=0.968, test=0.649) total time=   0.1s
[CV 2/5] END ..max_depth=26;, score=(train=0.962, test=0.646) total time=   0.1s
[CV 4/5] END ..max_depth=26;, score=(train=0.959, test=0.652) total time=   0.1s
[CV 4/5] END ..max_depth=27;, score=(train=0.964, test=0.650) total time=   0.1s
[CV 1/5] END ..max_depth=27;, score=(train=0.963, test=0.657) total time=   0.1s
[CV 1/5] END ..max_depth=28;, score=(train=0.968, test=0.655) total time=   0.1s
[CV 3/5] END ..max_depth=28;, score=(train=0.975, test=0.644) total time=   0.1s
[CV 5/5] END ..max_depth=28;, score=(train=0.973, test=0.636) total time=   0.1s
[CV 3/5] END ..max_depth=27;, score=(train=0.969, test=0.644) total time=   0.1s
[CV 2/5] END ..max_depth=29;, score=(train=0.978, test=0.640) total time=   0.1s
[CV 3/5] END ..max_depth=29;, score=(train=0.980, test=0.645) total time=   0.1s
[CV 5/5] END ..max_depth=27;, score=(train=0.966, test=0.644) total time=   0.1s
[CV 2/5] END ..max_depth=28;, score=(train=0.974, test=0.643) total time=   0.1s
[CV 4/5] END ..max_depth=29;, score=(train=0.975, test=0.647) total time=   0.2s
[CV 1/5] END ..max_depth=29;, score=(train=0.972, test=0.657) total time=   0.1s
[CV 4/5] END ..max_depth=28;, score=(train=0.969, test=0.643) total time=   0.2s
[CV 5/5] END ..max_depth=29;, score=(train=0.979, test=0.640) total time=   0.1s
[CV 1/5] END ..max_depth=30;, score=(train=0.976, test=0.653) total time=   0.2s
[CV 2/5] END ..max_depth=30;, score=(train=0.982, test=0.640) total time=   0.2s
[CV 3/5] END ..max_depth=30;, score=(train=0.985, test=0.643) total time=   0.1s
[CV 4/5] END ..max_depth=30;, score=(train=0.979, test=0.644) total time=   0.2s
[CV 5/5] END ..max_depth=30;, score=(train=0.983, test=0.638) total time=   0.2s
[CV 1/5] END ..max_depth=31;, score=(train=0.979, test=0.651) total time=   0.2s
[CV 3/5] END ..max_depth=31;, score=(train=0.987, test=0.644) total time=   0.1s
[CV 2/5] END ..max_depth=31;, score=(train=0.986, test=0.638) total time=   0.2s
[CV 4/5] END ..max_depth=31;, score=(train=0.982, test=0.645) total time=   0.1s
[CV 5/5] END ..max_depth=31;, score=(train=0.987, test=0.634) total time=   0.1s
CPU times: user 272 ms, sys: 196 ms, total: 468 ms
Wall time: 3.55 s

# plot training & testing scores
train_dt_scores = DecisionTree_GSCV.cv_results_['mean_train_score']
test_dt_scores = DecisionTree_GSCV.cv_results_['mean_test_score']

plt.plot(train_dt_scores, "g.--")
plt.plot(test_dt_scores, "g.-")
plt.ylim(0.4, 1.05)
plt.xticks(range(30), range(2, 32))
plt.legend(["DT training score", "DT test score"])
plt.axvline(np.argmax(test_dt_scores), linestyle="dotted", color="red")
plt.annotate(np.max(test_dt_scores).round(4), (np.argmax(test_dt_scores), np.max(test_dt_scores)), xycoords="data",
                 xytext=(50, 25), textcoords="offset pixels", arrowprops=dict(facecolor="black", shrink=0.1), fontsize=10,
                 horizontalalignment="center", verticalalignment="top")
plt.show()

# accuracy scores
print(DecisionTree_model.get_params())
print(f"Training Score: {round(DecisionTree_model.score(x_train, y_train),4)}")
print(f"Testing Score: {round(DecisionTree_model.score(x_test, y_test),4)}")

{'ccp_alpha': 0.0, 'class_weight': None, 'criterion': 'gini', 'max_depth': 6, 'max_features': None, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 1, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'random_state': 8017, 'splitter': 'best'}
Training Score: 0.7354
Testing Score: 0.7276

# feature importances
d = {'feature importance':list(DecisionTree_model.feature_importances_)}
table = pd.DataFrame(d, index=x_train.columns)

display(  table.sort_values('feature importance', ascending=False)  )

	feature importance
ap_hi	0.7633
age	0.1238
cholesterol	0.0733
weight	0.0158
gluc	0.0076
ap_lo	0.0059
height	0.0052
active	0.0043
smoke	0.0008
gender	0.0000
alco	0.0000

Logistic Regression

%%time

# fitting
Logistic_model =  LogisticRegressionCV(Cs = 50, cv = 5, random_state=8017)
Logistic_model.fit(x_train, y_train)
print(Logistic_model.get_params())

{'Cs': 50, 'class_weight': None, 'cv': 5, 'dual': False, 'fit_intercept': True, 'intercept_scaling': 1.0, 'l1_ratios': None, 'max_iter': 100, 'multi_class': 'auto', 'n_jobs': None, 'penalty': 'l2', 'random_state': 8017, 'refit': True, 'scoring': None, 'solver': 'lbfgs', 'tol': 0.0001, 'verbose': 0}
CPU times: user 25 s, sys: 676 ms, total: 25.7 s
Wall time: 3.47 s

# regularaization candidates
print('Candidates of Regularization Parameter C:')
print(Logistic_model.Cs_, '\n')

# accuracy scores
print(f"Training Score: {round(Logistic_model.score(x_train, y_train),4)}")
print(f"Testing Score: {round(Logistic_model.score(x_test, y_test),4)}")

Candidates of Regularization Parameter C:
[1.00000000e-04 1.45634848e-04 2.12095089e-04 3.08884360e-04
 4.49843267e-04 6.55128557e-04 9.54095476e-04 1.38949549e-03
 2.02358965e-03 2.94705170e-03 4.29193426e-03 6.25055193e-03
 9.10298178e-03 1.32571137e-02 1.93069773e-02 2.81176870e-02
 4.09491506e-02 5.96362332e-02 8.68511374e-02 1.26485522e-01
 1.84206997e-01 2.68269580e-01 3.90693994e-01 5.68986603e-01
 8.28642773e-01 1.20679264e+00 1.75751062e+00 2.55954792e+00
 3.72759372e+00 5.42867544e+00 7.90604321e+00 1.15139540e+01
 1.67683294e+01 2.44205309e+01 3.55648031e+01 5.17947468e+01
 7.54312006e+01 1.09854114e+02 1.59985872e+02 2.32995181e+02
 3.39322177e+02 4.94171336e+02 7.19685673e+02 1.04811313e+03
 1.52641797e+03 2.22299648e+03 3.23745754e+03 4.71486636e+03
 6.86648845e+03 1.00000000e+04] 

Training Score: 0.7276
Testing Score: 0.7274

# fitted parameters
print(f'Best Regularization Parameter C = {round(Logistic_model.C_[0],4)}')
print(f'intercept = {round(Logistic_model.intercept_[0],4)}')
d = {'estimates' : list(Logistic_model.coef_[0]),
     'absolute' : np.abs(list(Logistic_model.coef_[0]))
    }
table = pd.DataFrame(d, index=x_train.columns)

display(  table.sort_values('absolute', ascending=False).drop(columns='absolute')  )

Best Regularization Parameter C = 0.1842
intercept = -0.5036

	estimates
ap_hi	1.0920
age	0.5223
cholesterol	0.5054
active	-0.2129
alco	-0.1911
weight	0.1875
gluc	-0.1457
ap_lo	0.1424
smoke	-0.1324
gender	-0.0292
height	-0.0208

Cluster Analysis

Using the two most important features: age and ap_hi

# sample the first 2500 records only due to computation limit
sX = x_train[['age', 'ap_hi']][0:2500].to_numpy()
sY = y_train[0:2500].to_numpy()

K-means

# function to plot decision boundary
def plot_decision_boundary(x, y, model, title):
    
    h = 0.02
    x_min, x_max = x[:, 0].min()-0.1, x[:, 0].max() +0.1
    y_min, y_max = x[:, 1].min()-0.1, x[:, 1].max() +0.1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))

    # Obtain labels for each point in mesh. Use last trained model.
    Z = model.predict(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(xx.shape)
    plt.clf()
    plt.imshow(Z, interpolation='nearest',
               extent=(xx.min(), xx.max(), yy.min(), yy.max()),
               cmap=plt.cm.Paired,
               aspect='auto', origin='lower')

    plt.scatter(x[:, 0:1], x[:, 1:2], c=y, edgecolors='k')
    plt.title(title, fontsize = 20)

# K-means random
kmean1 = KMeans(n_clusters=2, init='random', random_state=0)
kmean1.fit(sX)
plot_decision_boundary(sX, sY, kmean1, "Prediction Boundary of K-Means")
plt.plot(kmean1.cluster_centers_[:, 0], kmean1.cluster_centers_[:, 1], '*', markersize=20, color="red")

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/cluster/_kmeans.py:1416: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning
  super()._check_params_vs_input(X, default_n_init=10)

[<matplotlib.lines.Line2D at 0x3013a0d60>]

# K-means++
kmean2 = KMeans(n_clusters=2, init='k-means++', random_state=0)
kmean2.fit(sX)
plot_decision_boundary(sX, sY, kmean2, "Prediction Boundary of K-Means ++")
plt.plot(kmean2.cluster_centers_[:, 0], kmean2.cluster_centers_[:,1], '*', markersize=20, color="red")

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/cluster/_kmeans.py:1416: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning
  super()._check_params_vs_input(X, default_n_init=10)

[<matplotlib.lines.Line2D at 0x305668250>]

# training accuracy scores
kmeans1_pred = kmean1.predict(sX) # K-means random
kmeans2_pred = kmean2.predict(sX) # K-means++
print('K-means(random) training accuracy: ', accuracy_score(sY, kmeans1_pred))
print('K-means++ training accuracy: ', accuracy_score(sY, kmeans2_pred), '\n')

# testing accuracy scores
kmeans1_pred_test = kmean1.predict(x_test[['age','ap_hi']]) # K-means random
kmeans2_pred_test = kmean2.predict(x_test[['age','ap_hi']]) # K-means++
print('K-means(random) testing accuracy: ', accuracy_score(y_test, kmeans1_pred_test))
print('K-means++ testing accuracy: ', accuracy_score(y_test, kmeans2_pred_test))

K-means(random) training accuracy:  0.2932
K-means++ training accuracy:  0.706 

K-means(random) testing accuracy:  0.2944515190103693
K-means++ testing accuracy:  0.7051482626887393

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/base.py:458: UserWarning: X has feature names, but KMeans was fitted without feature names
  warnings.warn(
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/base.py:458: UserWarning: X has feature names, but KMeans was fitted without feature names
  warnings.warn(

Agglomerative Clustering

# function to plot dengrogram
def plot_dendrogram(model, **kwargs): # provided by Mathew Kallada. 

    # Children of hierarchical clustering
    children = model.children_

    # Distances between each pair of children
    # Since we don't have this information, we can use a uniform one for plotting
    distance = np.arange(children.shape[0])

    # The number of observations contained in each cluster level
    no_of_observations = np.arange(2, children.shape[0] + 2)

    # Create linkage matrix and then plot the dendrogram
    linkage_matrix = np.column_stack([children, distance, no_of_observations]).astype(float)
    
    sch.dendrogram(linkage_matrix, **kwargs)

# ward's linkage and complete linkage
H_C_ward = AgglomerativeClustering(n_clusters=2) # default linkage is ward. 
H_C_complete = AgglomerativeClustering(n_clusters=2, linkage='complete')

# dendrogram (on 250 records only)
hc_ward_pred = H_C_ward.fit_predict(sX[0:250])
hc_complete_pred = H_C_complete.fit_predict(sX[0:250])

fig = plt.figure(figsize=(25, 10))
ax = fig.add_subplot(1, 2, 1)
plot_dendrogram(H_C_ward)
ax.set_title('Linkage method is ward')

ax = fig.add_subplot(1, 2, 2)
Z2 = plot_dendrogram(H_C_complete)
ax.set_title('Linkage method is complete')
plt.show()

# accuracy score (on 2500 records)
hc_ward_pred = H_C_ward.fit_predict(sX)
hc_complete_pred = H_C_complete.fit_predict(sX)

print("ward's linkage training accuracy: ", accuracy_score(sY, hc_ward_pred))
print('complete linkage training accuracy: ', accuracy_score(sY, hc_complete_pred))

ward's linkage training accuracy:  0.6836
complete linkage training accuracy:  0.5152

DBSCAN

# DBSCAN
dbscan = DBSCAN(eps=0.26, min_samples=20)
dbscan_pred = dbscan.fit_predict(sX)

# Number of clusters in labels, ignoring noise if present.
n_clusters_ = len(set(dbscan_pred)) - (1 if -1 in dbscan_pred else 0)
n_noise_ = list(dbscan_pred).count(-1)

print('Estimated number of clusters: %d' % n_clusters_)
print('Estimated number of noise points: %d' % n_noise_)

Estimated number of clusters: 2
Estimated number of noise points: 125

# plot
plt.scatter(x=sX[:,0], y=sX[:,1], c=dbscan_pred, edgecolors='k')
plt.show()

# accuracy score
print("DBSCAN training accuracy: ", accuracy_score(sY, dbscan_pred))

DBSCAN training accuracy:  0.4448

Gaussian Mixture

# Gaussian Mixture
gmm = GaussianMixture(n_components=2, covariance_type='full', max_iter=20, random_state=8017) 
gmm.fit(sX)
plot_decision_boundary(sX, sY, gmm, "Gaussian Mixture")

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/mixture/_base.py:268: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.
  warnings.warn(

# training accuracy scores
gmm_pred = gmm.predict(sX)
print('Gaussian Mixture Model training accuracy: ', accuracy_score(sY, gmm_pred), '\n')

# testing accuracy scores
gmm_pred_test = gmm.predict(x_test[['age','ap_hi']])
print('Gaussian Mixture Model testing accuracy: ', accuracy_score(y_test, gmm_pred_test))

Gaussian Mixture Model training accuracy:  0.3324 

Gaussian Mixture Model testing accuracy:  0.3342186647262143

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/base.py:458: UserWarning: X has feature names, but GaussianMixture was fitted without feature names
  warnings.warn(

Clustering Performance

# function to calculate entropy score
def get_entropy(y, pred, n_class):
    p = np.zeros((n_class, n_class))
    tb = contingency_matrix(y, pred)
    for i in range(n_class):
        for j in range(n_class):
            p[i, j] = tb[i, j]/np.sum(tb[i, :])
            
    E = np.zeros((n_class, 1))
    for i in range(n_class):
        for j in range(n_class):
            if (p[i, j] != 0):
                E[i] = E[i] - p[i, j] * np.log(p[i, j])
    Entropy = np.dot(np.sum(tb, 1) / np.sum(tb), E)
    return Entropy

# accuracy scores of all clustering
result = pd.DataFrame({'Model':['K-means (Random)','K-means (K-means++)','Dendrogram (Ward)','Dendrogram (Complete)','DBSCAN','Gaussian Mixture Model'],
                       'Training Accuracy': [accuracy_score(sY, kmeans1_pred),
                                             accuracy_score(sY, kmeans2_pred),
                                             accuracy_score(sY, hc_ward_pred),
                                             accuracy_score(sY, hc_complete_pred),
                                             accuracy_score(sY, dbscan_pred),
                                             accuracy_score(sY, gmm_pred)],
                       
                       # external measurement
                       'Entropy': [get_entropy(sY, kmeans1_pred, 2)[0], 
                                  get_entropy(sY, kmeans2_pred, 2)[0], 
                                  get_entropy(sY, hc_ward_pred, 2)[0], 
                                  get_entropy(sY, hc_complete_pred, 2)[0],
                                  get_entropy(sY, dbscan_pred, 2)[0], 
                                  get_entropy(sY, gmm_pred, 2)[0]],
                       'Adjusted Rand Index': [adjusted_rand_score(sY, kmeans1_pred), 
                                              adjusted_rand_score(sY, kmeans2_pred),
                                              adjusted_rand_score(sY, hc_ward_pred), 
                                              adjusted_rand_score(sY, hc_complete_pred), 
                                              adjusted_rand_score(sY, dbscan_pred), 
                                              adjusted_rand_score(sY, gmm_pred)],
                       
                       # internal measurement
                       'Silhouette Coefficient': [silhouette_score(sX, kmeans1_pred),
                                                 silhouette_score(sX, kmeans2_pred), 
                                                 silhouette_score(sX, hc_ward_pred),
                                                 silhouette_score(sX, hc_complete_pred),
                                                 silhouette_score(sX, dbscan_pred),
                                                 silhouette_score(sX, gmm_pred)],
                       'Calinski Harabasz Score': [calinski_harabasz_score(sX, kmeans1_pred),  #Ratio of between-cluster dispersion to within-cluster dispersion
                                                   calinski_harabasz_score(sX, kmeans2_pred), 
                                                   calinski_harabasz_score(sX, hc_ward_pred),
                                                   calinski_harabasz_score(sX, hc_complete_pred),
                                                   calinski_harabasz_score(sX, dbscan_pred),
                                                   calinski_harabasz_score(sX, gmm_pred)]
                      })
result

	Model	Training Accuracy	Entropy	Adjusted Rand Index	Silhouette Coefficient	Calinski Harabasz Score
0	K-means (Random)	0.2932	0.5345	0.1706	0.3948	1,715.3830
1	K-means (K-means++)	0.7060	0.5346	0.1693	0.3951	1,715.3924
2	Dendrogram (Ward)	0.6836	0.5200	0.1343	0.3933	1,591.1868
3	Dendrogram (Complete)	0.5152	0.1542	-0.0005	0.4433	472.9488
4	DBSCAN	0.4448	0.2202	0.0019	0.2206	120.8891
5	Gaussian Mixture Model	0.3324	0.6225	0.1120	0.2794	624.3443

K-means has the highest training accuracy.

Dendrogram (Complete) works the best in entropy, while K-means (Random) works the best on adjusted rand index.

It can be observed that Dendrogram (Complete) works the best in silhouette score, while K-means (K-means++) works the best regarding calinski harabasz score.

Ensemble Methods

Bagging Classifier

%%time

# parameters candidates
parameters = {'base_estimator__max_depth': [4,6,8,12,24],
              'n_estimators': [20, 50, 100, 200]}

# fitting
Bagging_GSCV = GridSearchCV(BaggingClassifier(DecisionTreeClassifier(), random_state=8017), 
                            parameters, n_jobs=-1, verbose=3, return_train_score=True)
Bagging_GSCV.fit(x_train, y_train)
Bagging_model = Bagging_GSCV.best_estimator_
Bagging_model

Fitting 5 folds for each of 20 candidates, totalling 100 fits

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=4, n_estimators=20;, score=(train=0.729, test=0.727) total time=   0.5s
[CV 1/5] END base_estimator__max_depth=4, n_estimators=20;, score=(train=0.728, test=0.733) total time=   0.5s
[CV 3/5] END base_estimator__max_depth=4, n_estimators=20;, score=(train=0.728, test=0.734) total time=   0.5s
[CV 4/5] END base_estimator__max_depth=4, n_estimators=20;, score=(train=0.731, test=0.720) total time=   0.5s
[CV 5/5] END base_estimator__max_depth=4, n_estimators=20;, score=(train=0.731, test=0.730) total time=   0.5s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=4, n_estimators=50;, score=(train=0.728, test=0.734) total time=   1.2s
[CV 2/5] END base_estimator__max_depth=4, n_estimators=50;, score=(train=0.731, test=0.726) total time=   1.2s
[CV 3/5] END base_estimator__max_depth=4, n_estimators=50;, score=(train=0.728, test=0.734) total time=   1.3s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=4, n_estimators=50;, score=(train=0.731, test=0.729) total time=   1.2s
[CV 4/5] END base_estimator__max_depth=4, n_estimators=50;, score=(train=0.731, test=0.720) total time=   1.2s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=4, n_estimators=100;, score=(train=0.729, test=0.734) total time=   2.3s
[CV 1/5] END base_estimator__max_depth=4, n_estimators=100;, score=(train=0.728, test=0.733) total time=   2.4s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=4, n_estimators=100;, score=(train=0.729, test=0.726) total time=   2.7s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=4, n_estimators=100;, score=(train=0.731, test=0.720) total time=   2.5s
[CV 5/5] END base_estimator__max_depth=4, n_estimators=100;, score=(train=0.731, test=0.730) total time=   2.4s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=6, n_estimators=20;, score=(train=0.737, test=0.739) total time=   0.7s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=6, n_estimators=20;, score=(train=0.738, test=0.738) total time=   0.7s
[CV 2/5] END base_estimator__max_depth=6, n_estimators=20;, score=(train=0.738, test=0.730) total time=   0.8s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=6, n_estimators=20;, score=(train=0.740, test=0.725) total time=   0.7s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=6, n_estimators=20;, score=(train=0.739, test=0.731) total time=   0.7s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=6, n_estimators=50;, score=(train=0.737, test=0.741) total time=   1.7s
[CV 1/5] END base_estimator__max_depth=4, n_estimators=200;, score=(train=0.728, test=0.733) total time=   5.0s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=6, n_estimators=50;, score=(train=0.738, test=0.729) total time=   1.8s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=4, n_estimators=200;, score=(train=0.729, test=0.734) total time=   4.9s
[CV 2/5] END base_estimator__max_depth=4, n_estimators=200;, score=(train=0.729, test=0.727) total time=   5.0s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=6, n_estimators=50;, score=(train=0.738, test=0.738) total time=   1.9s
[CV 5/5] END base_estimator__max_depth=4, n_estimators=200;, score=(train=0.730, test=0.730) total time=   5.2s
[CV 4/5] END base_estimator__max_depth=6, n_estimators=50;, score=(train=0.740, test=0.726) total time=   1.9s
[CV 4/5] END base_estimator__max_depth=4, n_estimators=200;, score=(train=0.731, test=0.720) total time=   5.2s
[CV 5/5] END base_estimator__max_depth=6, n_estimators=50;, score=(train=0.739, test=0.730) total time=   1.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=6, n_estimators=100;, score=(train=0.737, test=0.743) total time=   3.6s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=6, n_estimators=100;, score=(train=0.738, test=0.729) total time=   3.6s
[CV 3/5] END base_estimator__max_depth=6, n_estimators=100;, score=(train=0.738, test=0.739) total time=   3.6s
[CV 4/5] END base_estimator__max_depth=6, n_estimators=100;, score=(train=0.740, test=0.724) total time=   3.5s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=8, n_estimators=20;, score=(train=0.750, test=0.741) total time=   0.9s
[CV 2/5] END base_estimator__max_depth=8, n_estimators=20;, score=(train=0.751, test=0.728) total time=   0.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=6, n_estimators=100;, score=(train=0.740, test=0.729) total time=   3.5s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=8, n_estimators=20;, score=(train=0.753, test=0.725) total time=   0.8s
[CV 3/5] END base_estimator__max_depth=8, n_estimators=20;, score=(train=0.751, test=0.737) total time=   0.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=8, n_estimators=20;, score=(train=0.751, test=0.731) total time=   0.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=8, n_estimators=50;, score=(train=0.749, test=0.743) total time=   2.3s
[CV 2/5] END base_estimator__max_depth=8, n_estimators=50;, score=(train=0.752, test=0.728) total time=   2.3s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=8, n_estimators=50;, score=(train=0.752, test=0.737) total time=   2.2s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=6, n_estimators=200;, score=(train=0.736, test=0.742) total time=   7.1s
[CV 2/5] END base_estimator__max_depth=6, n_estimators=200;, score=(train=0.738, test=0.729) total time=   7.1s
[CV 3/5] END base_estimator__max_depth=6, n_estimators=200;, score=(train=0.738, test=0.738) total time=   7.1s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=8, n_estimators=50;, score=(train=0.753, test=0.726) total time=   2.2s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=8, n_estimators=50;, score=(train=0.751, test=0.730) total time=   2.3s
[CV 4/5] END base_estimator__max_depth=6, n_estimators=200;, score=(train=0.740, test=0.725) total time=   7.0s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=6, n_estimators=200;, score=(train=0.740, test=0.730) total time=   7.1s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=8, n_estimators=100;, score=(train=0.750, test=0.743) total time=   4.5s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=8, n_estimators=100;, score=(train=0.752, test=0.729) total time=   4.5s
[CV 4/5] END base_estimator__max_depth=8, n_estimators=100;, score=(train=0.754, test=0.725) total time=   4.5s
[CV 3/5] END base_estimator__max_depth=8, n_estimators=100;, score=(train=0.752, test=0.739) total time=   4.5s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=12, n_estimators=20;, score=(train=0.800, test=0.736) total time=   1.2s
[CV 5/5] END base_estimator__max_depth=8, n_estimators=100;, score=(train=0.753, test=0.730) total time=   4.4s
[CV 2/5] END base_estimator__max_depth=12, n_estimators=20;, score=(train=0.798, test=0.722) total time=   1.3s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=12, n_estimators=20;, score=(train=0.801, test=0.732) total time=   1.2s
[CV 4/5] END base_estimator__max_depth=12, n_estimators=20;, score=(train=0.799, test=0.722) total time=   1.3s
[CV 5/5] END base_estimator__max_depth=12, n_estimators=20;, score=(train=0.801, test=0.726) total time=   1.3s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=12, n_estimators=50;, score=(train=0.801, test=0.723) total time=   3.0s
[CV 1/5] END base_estimator__max_depth=12, n_estimators=50;, score=(train=0.801, test=0.738) total time=   3.1s
[CV 3/5] END base_estimator__max_depth=12, n_estimators=50;, score=(train=0.802, test=0.733) total time=   3.1s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=8, n_estimators=200;, score=(train=0.750, test=0.741) total time=   9.0s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=8, n_estimators=200;, score=(train=0.752, test=0.730) total time=   8.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=8, n_estimators=200;, score=(train=0.751, test=0.739) total time=   8.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=8, n_estimators=200;, score=(train=0.755, test=0.726) total time=   8.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=12, n_estimators=50;, score=(train=0.802, test=0.723) total time=   3.0s
[CV 5/5] END base_estimator__max_depth=8, n_estimators=200;, score=(train=0.753, test=0.730) total time=   8.7s
[CV 5/5] END base_estimator__max_depth=12, n_estimators=50;, score=(train=0.803, test=0.727) total time=   3.1s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=12, n_estimators=100;, score=(train=0.801, test=0.737) total time=   6.4s
[CV 2/5] END base_estimator__max_depth=12, n_estimators=100;, score=(train=0.803, test=0.723) total time=   6.2s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=12, n_estimators=100;, score=(train=0.803, test=0.735) total time=   6.2s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=12, n_estimators=100;, score=(train=0.802, test=0.722) total time=   6.3s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=24, n_estimators=20;, score=(train=0.964, test=0.712) total time=   1.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=12, n_estimators=100;, score=(train=0.804, test=0.727) total time=   6.3s
[CV 2/5] END base_estimator__max_depth=24, n_estimators=20;, score=(train=0.967, test=0.706) total time=   1.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=24, n_estimators=20;, score=(train=0.972, test=0.707) total time=   2.0s
[CV 4/5] END base_estimator__max_depth=24, n_estimators=20;, score=(train=0.966, test=0.702) total time=   1.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=24, n_estimators=20;, score=(train=0.968, test=0.703) total time=   1.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=24, n_estimators=50;, score=(train=0.972, test=0.718) total time=   4.7s
[CV 2/5] END base_estimator__max_depth=24, n_estimators=50;, score=(train=0.974, test=0.709) total time=   4.6s
[CV 2/5] END base_estimator__max_depth=12, n_estimators=200;, score=(train=0.803, test=0.725) total time=  12.7s
[CV 1/5] END base_estimator__max_depth=12, n_estimators=200;, score=(train=0.802, test=0.737) total time=  12.7s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=12, n_estimators=200;, score=(train=0.804, test=0.736) total time=  12.7s
[CV 3/5] END base_estimator__max_depth=24, n_estimators=50;, score=(train=0.978, test=0.712) total time=   4.8s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=12, n_estimators=200;, score=(train=0.803, test=0.723) total time=  12.8s
[CV 5/5] END base_estimator__max_depth=12, n_estimators=200;, score=(train=0.805, test=0.727) total time=  12.8s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=24, n_estimators=50;, score=(train=0.976, test=0.711) total time=   4.8s
[CV 5/5] END base_estimator__max_depth=24, n_estimators=50;, score=(train=0.976, test=0.709) total time=   4.8s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=24, n_estimators=100;, score=(train=0.975, test=0.719) total time=   9.4s
[CV 2/5] END base_estimator__max_depth=24, n_estimators=100;, score=(train=0.978, test=0.714) total time=   9.5s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=24, n_estimators=100;, score=(train=0.978, test=0.716) total time=   9.6s
[CV 4/5] END base_estimator__max_depth=24, n_estimators=100;, score=(train=0.977, test=0.711) total time=   9.7s
[CV 5/5] END base_estimator__max_depth=24, n_estimators=100;, score=(train=0.978, test=0.713) total time=   9.1s
[CV 1/5] END base_estimator__max_depth=24, n_estimators=200;, score=(train=0.976, test=0.720) total time=  16.8s
[CV 2/5] END base_estimator__max_depth=24, n_estimators=200;, score=(train=0.979, test=0.713) total time=  16.4s
[CV 3/5] END base_estimator__max_depth=24, n_estimators=200;, score=(train=0.980, test=0.717) total time=  16.6s
[CV 4/5] END base_estimator__max_depth=24, n_estimators=200;, score=(train=0.977, test=0.712) total time=  15.7s
[CV 5/5] END base_estimator__max_depth=24, n_estimators=200;, score=(train=0.980, test=0.715) total time=  15.7s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_search.py:930: FutureWarning: Parameter 'base_estimator' of BaggingClassifier is deprecated in favor of 'estimator'. See BaggingClassifier's docstring for more details.
  self.best_estimator_ = clone(base_estimator).set_params(

CPU times: user 8.61 s, sys: 235 ms, total: 8.85 s
Wall time: 1min 19s

BaggingClassifier(estimator=DecisionTreeClassifier(max_depth=8),
                  n_estimators=200, random_state=8017)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

train_bagging_scores = Bagging_GSCV.cv_results_['mean_train_score']
test_bagging_scores = Bagging_GSCV.cv_results_['mean_test_score']

#plt.plot(test_dt_scores, 'go-')
plt.plot(train_bagging_scores, 'ro--')
plt.plot(test_bagging_scores, 'ro-')
plt.ylim(0.4, 1.05)
plt.xticks(range(20), range(2, 22))
plt.legend(["Bagging training score", "Bagging test score"])
plt.axvline(np.argmax(test_bagging_scores), linestyle="dotted", color="red")
plt.annotate(np.max(test_bagging_scores).round(4), (np.argmax(test_bagging_scores), np.max(test_bagging_scores)), xycoords="data",
                 xytext=(-40, 30), textcoords="offset pixels", arrowprops=dict(facecolor="black", shrink=0.1), fontsize=10,
                 horizontalalignment="center", verticalalignment="top")
plt.show()

# accuracy scores
print(Bagging_model.get_params())
print(f"Training Score: {round(Bagging_model.score(x_train, y_train),4)}")
print(f"Testing Score: {round(Bagging_model.score(x_test, y_test),4)}")

{'base_estimator': 'deprecated', 'bootstrap': True, 'bootstrap_features': False, 'estimator__ccp_alpha': 0.0, 'estimator__class_weight': None, 'estimator__criterion': 'gini', 'estimator__max_depth': 8, 'estimator__max_features': None, 'estimator__max_leaf_nodes': None, 'estimator__min_impurity_decrease': 0.0, 'estimator__min_samples_leaf': 1, 'estimator__min_samples_split': 2, 'estimator__min_weight_fraction_leaf': 0.0, 'estimator__random_state': None, 'estimator__splitter': 'best', 'estimator': DecisionTreeClassifier(max_depth=8), 'max_features': 1.0, 'max_samples': 1.0, 'n_estimators': 200, 'n_jobs': None, 'oob_score': False, 'random_state': 8017, 'verbose': 0, 'warm_start': False}
Training Score: 0.7496
Testing Score: 0.7328

Random Forest Classifier

%%time

# parameters candidates
parameters = {'n_estimators': [20, 50, 100, 200],
              'max_depth':[6,8,12,24,48]}

# fitting
RandomForest_GSCV = GridSearchCV(RandomForestClassifier(random_state=8017), 
                                 parameters, n_jobs=-1, verbose=3)
RandomForest_GSCV.fit(x_train, y_train)
RandomForest_model = RandomForest_GSCV.best_estimator_
RandomForest_model

Fitting 5 folds for each of 20 candidates, totalling 100 fits
[CV 1/5] END ......max_depth=6, n_estimators=20;, score=0.736 total time=   0.2s
[CV 2/5] END ......max_depth=6, n_estimators=20;, score=0.731 total time=   0.2s
[CV 4/5] END ......max_depth=6, n_estimators=20;, score=0.722 total time=   0.2s
[CV 3/5] END ......max_depth=6, n_estimators=20;, score=0.734 total time=   0.3s
[CV 5/5] END ......max_depth=6, n_estimators=20;, score=0.727 total time=   0.3s
[CV 1/5] END ......max_depth=6, n_estimators=50;, score=0.735 total time=   0.6s
[CV 2/5] END ......max_depth=6, n_estimators=50;, score=0.729 total time=   0.6s
[CV 3/5] END ......max_depth=6, n_estimators=50;, score=0.734 total time=   0.6s
[CV 4/5] END ......max_depth=6, n_estimators=50;, score=0.723 total time=   0.6s
[CV 5/5] END ......max_depth=6, n_estimators=50;, score=0.727 total time=   0.7s
[CV 1/5] END .....max_depth=6, n_estimators=100;, score=0.736 total time=   1.2s
[CV 2/5] END .....max_depth=6, n_estimators=100;, score=0.728 total time=   1.3s
[CV 3/5] END .....max_depth=6, n_estimators=100;, score=0.733 total time=   1.3s
[CV 1/5] END ......max_depth=8, n_estimators=20;, score=0.737 total time=   0.3s
[CV 5/5] END .....max_depth=6, n_estimators=100;, score=0.727 total time=   1.3s
[CV 4/5] END .....max_depth=6, n_estimators=100;, score=0.725 total time=   1.3s
[CV 3/5] END ......max_depth=8, n_estimators=20;, score=0.735 total time=   0.3s
[CV 2/5] END ......max_depth=8, n_estimators=20;, score=0.729 total time=   0.3s
[CV 4/5] END ......max_depth=8, n_estimators=20;, score=0.727 total time=   0.3s
[CV 5/5] END ......max_depth=8, n_estimators=20;, score=0.729 total time=   0.3s
[CV 1/5] END ......max_depth=8, n_estimators=50;, score=0.739 total time=   0.8s
[CV 1/5] END .....max_depth=6, n_estimators=200;, score=0.734 total time=   2.4s
[CV 2/5] END ......max_depth=8, n_estimators=50;, score=0.730 total time=   0.9s
[CV 2/5] END .....max_depth=6, n_estimators=200;, score=0.728 total time=   2.4s
[CV 3/5] END ......max_depth=8, n_estimators=50;, score=0.735 total time=   0.8s
[CV 3/5] END .....max_depth=6, n_estimators=200;, score=0.733 total time=   2.5s
[CV 4/5] END ......max_depth=8, n_estimators=50;, score=0.726 total time=   0.8s
[CV 4/5] END .....max_depth=6, n_estimators=200;, score=0.725 total time=   2.4s
[CV 5/5] END ......max_depth=8, n_estimators=50;, score=0.731 total time=   0.8s
[CV 5/5] END .....max_depth=6, n_estimators=200;, score=0.728 total time=   2.5s
[CV 1/5] END .....max_depth=8, n_estimators=100;, score=0.739 total time=   1.5s
[CV 2/5] END .....max_depth=8, n_estimators=100;, score=0.731 total time=   1.5s
[CV 3/5] END .....max_depth=8, n_estimators=100;, score=0.736 total time=   1.5s
[CV 4/5] END .....max_depth=8, n_estimators=100;, score=0.726 total time=   1.6s
[CV 1/5] END .....max_depth=12, n_estimators=20;, score=0.738 total time=   0.4s
[CV 2/5] END .....max_depth=12, n_estimators=20;, score=0.727 total time=   0.4s
[CV 5/5] END .....max_depth=8, n_estimators=100;, score=0.730 total time=   1.6s
[CV 3/5] END .....max_depth=12, n_estimators=20;, score=0.735 total time=   0.4s
[CV 4/5] END .....max_depth=12, n_estimators=20;, score=0.725 total time=   0.4s
[CV 5/5] END .....max_depth=12, n_estimators=20;, score=0.732 total time=   0.4s
[CV 1/5] END .....max_depth=12, n_estimators=50;, score=0.739 total time=   1.0s
[CV 1/5] END .....max_depth=8, n_estimators=200;, score=0.739 total time=   3.0s
[CV 2/5] END .....max_depth=8, n_estimators=200;, score=0.731 total time=   3.0s
[CV 3/5] END .....max_depth=12, n_estimators=50;, score=0.738 total time=   1.0s
[CV 2/5] END .....max_depth=12, n_estimators=50;, score=0.729 total time=   1.2s
[CV 3/5] END .....max_depth=8, n_estimators=200;, score=0.736 total time=   3.2s
[CV 4/5] END .....max_depth=8, n_estimators=200;, score=0.727 total time=   3.1s
[CV 5/5] END .....max_depth=8, n_estimators=200;, score=0.731 total time=   3.0s
[CV 4/5] END .....max_depth=12, n_estimators=50;, score=0.726 total time=   1.1s
[CV 5/5] END .....max_depth=12, n_estimators=50;, score=0.735 total time=   1.1s
[CV 2/5] END ....max_depth=12, n_estimators=100;, score=0.729 total time=   2.1s
[CV 1/5] END ....max_depth=12, n_estimators=100;, score=0.740 total time=   2.1s
[CV 3/5] END ....max_depth=12, n_estimators=100;, score=0.737 total time=   2.1s
[CV 4/5] END ....max_depth=12, n_estimators=100;, score=0.726 total time=   2.1s
[CV 5/5] END ....max_depth=12, n_estimators=100;, score=0.733 total time=   2.1s
[CV 1/5] END .....max_depth=24, n_estimators=20;, score=0.727 total time=   0.7s
[CV 2/5] END .....max_depth=24, n_estimators=20;, score=0.712 total time=   0.7s
[CV 3/5] END .....max_depth=24, n_estimators=20;, score=0.722 total time=   0.6s
[CV 4/5] END .....max_depth=24, n_estimators=20;, score=0.706 total time=   0.6s
[CV 5/5] END .....max_depth=24, n_estimators=20;, score=0.716 total time=   0.6s
[CV 1/5] END ....max_depth=12, n_estimators=200;, score=0.741 total time=   4.2s
[CV 1/5] END .....max_depth=24, n_estimators=50;, score=0.725 total time=   1.6s
[CV 3/5] END ....max_depth=12, n_estimators=200;, score=0.741 total time=   4.2s
[CV 2/5] END .....max_depth=24, n_estimators=50;, score=0.720 total time=   1.6s
[CV 2/5] END ....max_depth=12, n_estimators=200;, score=0.729 total time=   4.3s
[CV 3/5] END .....max_depth=24, n_estimators=50;, score=0.723 total time=   1.6s
[CV 4/5] END ....max_depth=12, n_estimators=200;, score=0.726 total time=   4.2s
[CV 5/5] END ....max_depth=12, n_estimators=200;, score=0.733 total time=   4.3s
[CV 4/5] END .....max_depth=24, n_estimators=50;, score=0.716 total time=   1.7s
[CV 5/5] END .....max_depth=24, n_estimators=50;, score=0.718 total time=   1.7s
[CV 2/5] END ....max_depth=24, n_estimators=100;, score=0.722 total time=   3.3s
[CV 3/5] END ....max_depth=24, n_estimators=100;, score=0.728 total time=   3.3s
[CV 1/5] END ....max_depth=24, n_estimators=100;, score=0.727 total time=   3.4s
[CV 4/5] END ....max_depth=24, n_estimators=100;, score=0.719 total time=   3.3s
[CV 1/5] END .....max_depth=48, n_estimators=20;, score=0.712 total time=   0.7s
[CV 2/5] END .....max_depth=48, n_estimators=20;, score=0.702 total time=   0.7s
[CV 5/5] END ....max_depth=24, n_estimators=100;, score=0.720 total time=   3.2s
[CV 3/5] END .....max_depth=48, n_estimators=20;, score=0.711 total time=   0.7s
[CV 4/5] END .....max_depth=48, n_estimators=20;, score=0.702 total time=   0.7s
[CV 5/5] END .....max_depth=48, n_estimators=20;, score=0.704 total time=   0.7s
[CV 1/5] END .....max_depth=48, n_estimators=50;, score=0.715 total time=   1.7s
[CV 2/5] END .....max_depth=48, n_estimators=50;, score=0.705 total time=   1.7s
[CV 3/5] END .....max_depth=48, n_estimators=50;, score=0.715 total time=   1.8s
[CV 1/5] END ....max_depth=24, n_estimators=200;, score=0.727 total time=   6.6s
[CV 2/5] END ....max_depth=24, n_estimators=200;, score=0.723 total time=   6.3s
[CV 3/5] END ....max_depth=24, n_estimators=200;, score=0.728 total time=   6.4s
[CV 4/5] END .....max_depth=48, n_estimators=50;, score=0.710 total time=   1.8s
[CV 5/5] END .....max_depth=48, n_estimators=50;, score=0.711 total time=   1.7s
[CV 4/5] END ....max_depth=24, n_estimators=200;, score=0.718 total time=   6.3s
[CV 5/5] END ....max_depth=24, n_estimators=200;, score=0.721 total time=   6.5s
[CV 1/5] END ....max_depth=48, n_estimators=100;, score=0.718 total time=   3.5s
[CV 2/5] END ....max_depth=48, n_estimators=100;, score=0.706 total time=   3.4s
[CV 3/5] END ....max_depth=48, n_estimators=100;, score=0.716 total time=   3.4s
[CV 4/5] END ....max_depth=48, n_estimators=100;, score=0.712 total time=   3.4s
[CV 5/5] END ....max_depth=48, n_estimators=100;, score=0.715 total time=   3.5s
[CV 1/5] END ....max_depth=48, n_estimators=200;, score=0.719 total time=   6.3s
[CV 2/5] END ....max_depth=48, n_estimators=200;, score=0.711 total time=   5.9s
[CV 3/5] END ....max_depth=48, n_estimators=200;, score=0.718 total time=   5.9s
[CV 4/5] END ....max_depth=48, n_estimators=200;, score=0.710 total time=   5.7s
[CV 5/5] END ....max_depth=48, n_estimators=200;, score=0.715 total time=   5.7s
CPU times: user 4.07 s, sys: 167 ms, total: 4.24 s
Wall time: 33.1 s

RandomForestClassifier(max_depth=12, n_estimators=200, random_state=8017)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

# accuracy scores
print(RandomForest_model.get_params())
print(f"Training Score: {round(RandomForest_model.score(x_train, y_train),4)}")
print(f"Testing Score: {round(RandomForest_model.score(x_test, y_test),4)}")

{'bootstrap': True, 'ccp_alpha': 0.0, 'class_weight': None, 'criterion': 'gini', 'max_depth': 12, 'max_features': 'sqrt', 'max_leaf_nodes': None, 'max_samples': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 1, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'n_estimators': 200, 'n_jobs': None, 'oob_score': False, 'random_state': 8017, 'verbose': 0, 'warm_start': False}
Training Score: 0.7796
Testing Score: 0.7336

Adaboost

%%time

# parameters candidates
parameters = {'base_estimator__max_depth': [2,3,4,6,8,12],
              'n_estimators': [20, 50, 100, 200]}

# fitting
Adaboost_GSCV = GridSearchCV(AdaBoostClassifier(DecisionTreeClassifier(), random_state=8017), 
                             parameters, n_jobs=-1, verbose=3)
Adaboost_GSCV.fit(x_train, y_train)
Adaboost_model = Adaboost_GSCV.best_estimator_
Adaboost_model

Fitting 5 folds for each of 24 candidates, totalling 120 fits

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=2, n_estimators=20;, score=0.739 total time=   0.4s
[CV 2/5] END base_estimator__max_depth=2, n_estimators=20;, score=0.729 total time=   0.4s
[CV 3/5] END base_estimator__max_depth=2, n_estimators=20;, score=0.737 total time=   0.4s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=2, n_estimators=20;, score=0.731 total time=   0.4s
[CV 4/5] END base_estimator__max_depth=2, n_estimators=20;, score=0.724 total time=   0.4s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=2, n_estimators=50;, score=0.724 total time=   1.0s
[CV 5/5] END base_estimator__max_depth=2, n_estimators=50;, score=0.730 total time=   1.1s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=2, n_estimators=50;, score=0.728 total time=   1.1s
[CV 1/5] END base_estimator__max_depth=2, n_estimators=50;, score=0.739 total time=   1.1s
[CV 3/5] END base_estimator__max_depth=2, n_estimators=50;, score=0.735 total time=   1.1s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=2, n_estimators=100;, score=0.737 total time=   2.3s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=2, n_estimators=100;, score=0.730 total time=   2.1s
[CV 3/5] END base_estimator__max_depth=2, n_estimators=100;, score=0.736 total time=   2.1s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=2, n_estimators=100;, score=0.726 total time=   2.2s
[CV 5/5] END base_estimator__max_depth=2, n_estimators=100;, score=0.729 total time=   2.1s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=3, n_estimators=20;, score=0.739 total time=   0.6s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=3, n_estimators=20;, score=0.736 total time=   0.6s
[CV 2/5] END base_estimator__max_depth=3, n_estimators=20;, score=0.727 total time=   0.7s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=3, n_estimators=20;, score=0.725 total time=   0.6s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=3, n_estimators=20;, score=0.729 total time=   0.6s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=3, n_estimators=50;, score=0.737 total time=   1.5s
[CV 1/5] END base_estimator__max_depth=2, n_estimators=200;, score=0.736 total time=   4.1s
[CV 2/5] END base_estimator__max_depth=3, n_estimators=50;, score=0.725 total time=   1.4s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=2, n_estimators=200;, score=0.727 total time=   4.5s
[CV 3/5] END base_estimator__max_depth=2, n_estimators=200;, score=0.732 total time=   4.4s
[CV 3/5] END base_estimator__max_depth=3, n_estimators=50;, score=0.730 total time=   1.5s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=2, n_estimators=200;, score=0.721 total time=   4.3s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=3, n_estimators=50;, score=0.722 total time=   1.4s
[CV 5/5] END base_estimator__max_depth=3, n_estimators=50;, score=0.727 total time=   1.4s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=2, n_estimators=200;, score=0.725 total time=   4.3s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=3, n_estimators=100;, score=0.737 total time=   3.1s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=3, n_estimators=100;, score=0.719 total time=   3.1s
[CV 4/5] END base_estimator__max_depth=3, n_estimators=100;, score=0.715 total time=   3.2s
[CV 3/5] END base_estimator__max_depth=3, n_estimators=100;, score=0.726 total time=   3.3s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=3, n_estimators=100;, score=0.724 total time=   3.3s
[CV 1/5] END base_estimator__max_depth=4, n_estimators=20;, score=0.736 total time=   0.8s
[CV 2/5] END base_estimator__max_depth=4, n_estimators=20;, score=0.725 total time=   0.8s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=4, n_estimators=20;, score=0.736 total time=   0.8s
[CV 4/5] END base_estimator__max_depth=4, n_estimators=20;, score=0.725 total time=   0.8s
[CV 5/5] END base_estimator__max_depth=4, n_estimators=20;, score=0.727 total time=   0.8s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=4, n_estimators=50;, score=0.720 total time=   1.8s
[CV 1/5] END base_estimator__max_depth=4, n_estimators=50;, score=0.731 total time=   2.0s
[CV 3/5] END base_estimator__max_depth=4, n_estimators=50;, score=0.730 total time=   2.0s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=3, n_estimators=200;, score=0.713 total time=   6.2s
[CV 1/5] END base_estimator__max_depth=3, n_estimators=200;, score=0.726 total time=   6.3s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=3, n_estimators=200;, score=0.725 total time=   6.1s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=4, n_estimators=50;, score=0.719 total time=   2.0s
[CV 5/5] END base_estimator__max_depth=4, n_estimators=50;, score=0.725 total time=   2.0s
[CV 4/5] END base_estimator__max_depth=3, n_estimators=200;, score=0.709 total time=   6.0s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=3, n_estimators=200;, score=0.717 total time=   6.2s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=4, n_estimators=100;, score=0.719 total time=   3.8s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=4, n_estimators=100;, score=0.711 total time=   3.9s
[CV 3/5] END base_estimator__max_depth=4, n_estimators=100;, score=0.721 total time=   4.0s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=4, n_estimators=100;, score=0.713 total time=   3.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=6, n_estimators=20;, score=0.722 total time=   1.1s
[CV 2/5] END base_estimator__max_depth=6, n_estimators=20;, score=0.719 total time=   1.0s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=4, n_estimators=100;, score=0.716 total time=   3.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=6, n_estimators=20;, score=0.726 total time=   1.1s
[CV 4/5] END base_estimator__max_depth=6, n_estimators=20;, score=0.719 total time=   1.1s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=6, n_estimators=20;, score=0.712 total time=   1.1s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=6, n_estimators=50;, score=0.711 total time=   2.6s
[CV 2/5] END base_estimator__max_depth=6, n_estimators=50;, score=0.712 total time=   2.6s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=6, n_estimators=50;, score=0.710 total time=   2.6s
[CV 2/5] END base_estimator__max_depth=4, n_estimators=200;, score=0.704 total time=   7.7s
[CV 1/5] END base_estimator__max_depth=4, n_estimators=200;, score=0.704 total time=   7.8s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=4, n_estimators=200;, score=0.703 total time=   7.6s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=4, n_estimators=200;, score=0.700 total time=   7.4s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=4, n_estimators=200;, score=0.704 total time=   7.5s
[CV 4/5] END base_estimator__max_depth=6, n_estimators=50;, score=0.703 total time=   2.7s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=6, n_estimators=50;, score=0.707 total time=   2.6s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=6, n_estimators=100;, score=0.687 total time=   5.3s
[CV 2/5] END base_estimator__max_depth=6, n_estimators=100;, score=0.688 total time=   5.4s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=6, n_estimators=100;, score=0.689 total time=   5.4s
[CV 4/5] END base_estimator__max_depth=6, n_estimators=100;, score=0.687 total time=   5.3s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=6, n_estimators=100;, score=0.689 total time=   5.2s
[CV 1/5] END base_estimator__max_depth=8, n_estimators=20;, score=0.707 total time=   1.4s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=8, n_estimators=20;, score=0.710 total time=   1.4s
[CV 3/5] END base_estimator__max_depth=8, n_estimators=20;, score=0.711 total time=   1.6s
[CV 4/5] END base_estimator__max_depth=8, n_estimators=20;, score=0.702 total time=   1.7s
[CV 5/5] END base_estimator__max_depth=8, n_estimators=20;, score=0.704 total time=   1.6s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=8, n_estimators=50;, score=0.681 total time=   3.5s
[CV 1/5] END base_estimator__max_depth=8, n_estimators=50;, score=0.679 total time=   3.5s
[CV 3/5] END base_estimator__max_depth=8, n_estimators=50;, score=0.687 total time=   3.5s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=6, n_estimators=200;, score=0.668 total time=  10.9s
[CV 2/5] END base_estimator__max_depth=6, n_estimators=200;, score=0.662 total time=  10.8s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=6, n_estimators=200;, score=0.669 total time=  10.9s
[CV 5/5] END base_estimator__max_depth=8, n_estimators=50;, score=0.686 total time=   3.4s
[CV 4/5] END base_estimator__max_depth=8, n_estimators=50;, score=0.669 total time=   3.4s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=6, n_estimators=200;, score=0.664 total time=  10.9s
[CV 5/5] END base_estimator__max_depth=6, n_estimators=200;, score=0.658 total time=  10.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=8, n_estimators=100;, score=0.669 total time=   6.8s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=8, n_estimators=100;, score=0.660 total time=   6.6s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=8, n_estimators=100;, score=0.668 total time=   6.8s
[CV 4/5] END base_estimator__max_depth=8, n_estimators=100;, score=0.639 total time=   6.7s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=12, n_estimators=20;, score=0.666 total time=   2.0s
[CV 1/5] END base_estimator__max_depth=12, n_estimators=20;, score=0.672 total time=   2.0s
[CV 5/5] END base_estimator__max_depth=8, n_estimators=100;, score=0.659 total time=   6.6s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=12, n_estimators=20;, score=0.665 total time=   1.9s
[CV 4/5] END base_estimator__max_depth=12, n_estimators=20;, score=0.662 total time=   2.0s
[CV 5/5] END base_estimator__max_depth=12, n_estimators=20;, score=0.678 total time=   1.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=12, n_estimators=50;, score=0.670 total time=   4.8s
[CV 2/5] END base_estimator__max_depth=12, n_estimators=50;, score=0.665 total time=   4.8s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=12, n_estimators=50;, score=0.665 total time=   4.9s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=8, n_estimators=200;, score=0.662 total time=  13.8s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 3/5] END base_estimator__max_depth=8, n_estimators=200;, score=0.667 total time=  13.4s
[CV 2/5] END base_estimator__max_depth=8, n_estimators=200;, score=0.662 total time=  13.5s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=8, n_estimators=200;, score=0.658 total time=  13.5s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 5/5] END base_estimator__max_depth=8, n_estimators=200;, score=0.664 total time=  13.3s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=12, n_estimators=50;, score=0.669 total time=   4.9s
[CV 5/5] END base_estimator__max_depth=12, n_estimators=50;, score=0.666 total time=   5.0s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))
/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 1/5] END base_estimator__max_depth=12, n_estimators=100;, score=0.689 total time=   8.3s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 2/5] END base_estimator__max_depth=12, n_estimators=100;, score=0.684 total time=   8.5s
[CV 3/5] END base_estimator__max_depth=12, n_estimators=100;, score=0.684 total time=   8.3s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:717: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  estimator = estimator.set_params(**clone(parameters, safe=False))

[CV 4/5] END base_estimator__max_depth=12, n_estimators=100;, score=0.683 total time=   8.5s
[CV 5/5] END base_estimator__max_depth=12, n_estimators=100;, score=0.684 total time=   8.4s
[CV 1/5] END base_estimator__max_depth=12, n_estimators=200;, score=0.689 total time=  10.1s
[CV 2/5] END base_estimator__max_depth=12, n_estimators=200;, score=0.683 total time=  10.4s
[CV 3/5] END base_estimator__max_depth=12, n_estimators=200;, score=0.684 total time=  11.2s
[CV 4/5] END base_estimator__max_depth=12, n_estimators=200;, score=0.684 total time=   9.6s
[CV 5/5] END base_estimator__max_depth=12, n_estimators=200;, score=0.686 total time=   9.7s

/Users/vsaravind/.local/lib/python3.10/site-packages/sklearn/model_selection/_search.py:930: FutureWarning: Parameter 'base_estimator' of AdaBoostClassifier is deprecated in favor of 'estimator'. See AdaBoostClassifier's docstring for more details.
  self.best_estimator_ = clone(base_estimator).set_params(

CPU times: user 902 ms, sys: 262 ms, total: 1.16 s
Wall time: 1min 11s

AdaBoostClassifier(estimator=DecisionTreeClassifier(max_depth=2),
                   n_estimators=20, random_state=8017)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

# accuracy scores
print(Adaboost_model.get_params())
print(f"Training Score: {round(Adaboost_model.score(x_train, y_train),4)}")
print(f"Testing Score: {round(Adaboost_model.score(x_test, y_test),4)}")

{'algorithm': 'SAMME.R', 'base_estimator': 'deprecated', 'estimator__ccp_alpha': 0.0, 'estimator__class_weight': None, 'estimator__criterion': 'gini', 'estimator__max_depth': 2, 'estimator__max_features': None, 'estimator__max_leaf_nodes': None, 'estimator__min_impurity_decrease': 0.0, 'estimator__min_samples_leaf': 1, 'estimator__min_samples_split': 2, 'estimator__min_weight_fraction_leaf': 0.0, 'estimator__random_state': None, 'estimator__splitter': 'best', 'estimator': DecisionTreeClassifier(max_depth=2), 'learning_rate': 1.0, 'n_estimators': 20, 'random_state': 8017}
Training Score: 0.7335
Testing Score: 0.7315

Gradient Boosting

%%time

# parameters candidates
parameters = {'max_depth': [2,4,6,8,10,12],
              'n_estimators': [200],
              'learning_rate': [0.01]}

# fitting
GradientBoost_GSCV = GridSearchCV(GradientBoostingClassifier(random_state=8017), 
                             parameters, n_jobs=-1, verbose=3)
GradientBoost_GSCV.fit(x_train, y_train)
GradientBoost_model = GradientBoost_GSCV.best_estimator_
GradientBoost_model

Fitting 5 folds for each of 6 candidates, totalling 30 fits
[CV 1/5] END learning_rate=0.01, max_depth=2, n_estimators=200;, score=0.728 total time=   3.5s
[CV 3/5] END learning_rate=0.01, max_depth=2, n_estimators=200;, score=0.726 total time=   3.6s
[CV 2/5] END learning_rate=0.01, max_depth=2, n_estimators=200;, score=0.724 total time=   3.6s
[CV 4/5] END learning_rate=0.01, max_depth=2, n_estimators=200;, score=0.717 total time=   3.6s
[CV 5/5] END learning_rate=0.01, max_depth=2, n_estimators=200;, score=0.724 total time=   3.6s
[CV 2/5] END learning_rate=0.01, max_depth=4, n_estimators=200;, score=0.729 total time=   6.8s
[CV 1/5] END learning_rate=0.01, max_depth=4, n_estimators=200;, score=0.738 total time=   6.8s
[CV 3/5] END learning_rate=0.01, max_depth=4, n_estimators=200;, score=0.734 total time=   6.9s
[CV 5/5] END learning_rate=0.01, max_depth=4, n_estimators=200;, score=0.729 total time=   6.7s
[CV 4/5] END learning_rate=0.01, max_depth=4, n_estimators=200;, score=0.725 total time=   7.0s
[CV 1/5] END learning_rate=0.01, max_depth=6, n_estimators=200;, score=0.740 total time=   9.7s
[CV 2/5] END learning_rate=0.01, max_depth=6, n_estimators=200;, score=0.730 total time=   9.7s
[CV 3/5] END learning_rate=0.01, max_depth=6, n_estimators=200;, score=0.738 total time=   9.8s
[CV 4/5] END learning_rate=0.01, max_depth=6, n_estimators=200;, score=0.727 total time=   9.7s
[CV 5/5] END learning_rate=0.01, max_depth=6, n_estimators=200;, score=0.731 total time=   9.9s
[CV 1/5] END learning_rate=0.01, max_depth=8, n_estimators=200;, score=0.738 total time=  13.5s
[CV 2/5] END learning_rate=0.01, max_depth=8, n_estimators=200;, score=0.729 total time=  13.4s
[CV 3/5] END learning_rate=0.01, max_depth=8, n_estimators=200;, score=0.738 total time=  13.4s
[CV 5/5] END learning_rate=0.01, max_depth=8, n_estimators=200;, score=0.729 total time=  13.2s
[CV 4/5] END learning_rate=0.01, max_depth=8, n_estimators=200;, score=0.722 total time=  13.5s
[CV 1/5] END learning_rate=0.01, max_depth=10, n_estimators=200;, score=0.733 total time=  17.7s
[CV 2/5] END learning_rate=0.01, max_depth=10, n_estimators=200;, score=0.723 total time=  17.9s
[CV 3/5] END learning_rate=0.01, max_depth=10, n_estimators=200;, score=0.733 total time=  17.9s
[CV 4/5] END learning_rate=0.01, max_depth=10, n_estimators=200;, score=0.719 total time=  17.6s
[CV 5/5] END learning_rate=0.01, max_depth=10, n_estimators=200;, score=0.725 total time=  17.2s
[CV 1/5] END learning_rate=0.01, max_depth=12, n_estimators=200;, score=0.728 total time=  22.2s
[CV 3/5] END learning_rate=0.01, max_depth=12, n_estimators=200;, score=0.728 total time=  21.8s
[CV 2/5] END learning_rate=0.01, max_depth=12, n_estimators=200;, score=0.717 total time=  22.0s
[CV 4/5] END learning_rate=0.01, max_depth=12, n_estimators=200;, score=0.717 total time=  20.8s
[CV 5/5] END learning_rate=0.01, max_depth=12, n_estimators=200;, score=0.722 total time=  20.3s
CPU times: user 9.81 s, sys: 104 ms, total: 9.91 s
Wall time: 1min 4s

GradientBoostingClassifier(learning_rate=0.01, max_depth=6, n_estimators=200,
                           random_state=8017)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

# accuracy scores
print(GradientBoost_model.get_params())
print(f"Training Score: {round(GradientBoost_model.score(x_train, y_train),4)}")
print(f"Testing Score: {round(GradientBoost_model.score(x_test, y_test),4)}")

{'ccp_alpha': 0.0, 'criterion': 'friedman_mse', 'init': None, 'learning_rate': 0.01, 'loss': 'log_loss', 'max_depth': 6, 'max_features': None, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 1, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'n_estimators': 200, 'n_iter_no_change': None, 'random_state': 8017, 'subsample': 1.0, 'tol': 0.0001, 'validation_fraction': 0.1, 'verbose': 0, 'warm_start': False}
Training Score: 0.7438
Testing Score: 0.7332

Support Vector Machine

Linear SVC

%%time

# parameters candidates
parameters = {'C': np.logspace(-4, 4, 50)}

# fitting
LinearSVC_GSCV = GridSearchCV(LinearSVC(dual=False, random_state=8017), 
                              parameters, n_jobs=-1, verbose=3)
LinearSVC_GSCV.fit(x_train, y_train)
LinearSVC_model = LinearSVC_GSCV.best_estimator_
LinearSVC_model

Fitting 5 folds for each of 50 candidates, totalling 250 fits
[CV 3/5] END ..........................C=0.0001;, score=0.730 total time=   0.0s
[CV 1/5] END ..........................C=0.0001;, score=0.732 total time=   0.0s
[CV 2/5] END ..........................C=0.0001;, score=0.725 total time=   0.0s
[CV 4/5] END ..........................C=0.0001;, score=0.717 total time=   0.0s
[CV 5/5] END ..........................C=0.0001;, score=0.727 total time=   0.0s
[CV 1/5] END ..........C=0.00014563484775012445;, score=0.732 total time=   0.0s
[CV 2/5] END ..........C=0.00014563484775012445;, score=0.724 total time=   0.0s
[CV 3/5] END ..........C=0.00014563484775012445;, score=0.730 total time=   0.0s
[CV 5/5] END ..........C=0.00014563484775012445;, score=0.725 total time=   0.0s
[CV 4/5] END ..........C=0.00014563484775012445;, score=0.719 total time=   0.0s
[CV 1/5] END ..........C=0.00021209508879201905;, score=0.732 total time=   0.0s
[CV 2/5] END ..........C=0.00021209508879201905;, score=0.726 total time=   0.0s
[CV 3/5] END ..........C=0.00021209508879201905;, score=0.728 total time=   0.0s
[CV 4/5] END ..........C=0.00021209508879201905;, score=0.720 total time=   0.0s
[CV 5/5] END ..........C=0.00021209508879201905;, score=0.725 total time=   0.0s
[CV 1/5] END ..........C=0.00030888435964774815;, score=0.731 total time=   0.0s
[CV 2/5] END ..........C=0.00030888435964774815;, score=0.726 total time=   0.0s
[CV 4/5] END ..........C=0.00030888435964774815;, score=0.721 total time=   0.0s
[CV 3/5] END ...........C=0.0004498432668969444;, score=0.728 total time=   0.0s
[CV 1/5] END ...........C=0.0004498432668969444;, score=0.731 total time=   0.0s
[CV 5/5] END ...........C=0.0004498432668969444;, score=0.724 total time=   0.0s
[CV 2/5] END ...........C=0.0006551285568595509;, score=0.726 total time=   0.0s
[CV 3/5] END ..........C=0.00030888435964774815;, score=0.728 total time=   0.0s
[CV 5/5] END ..........C=0.00030888435964774815;, score=0.724 total time=   0.0s
[CV 4/5] END ...........C=0.0006551285568595509;, score=0.720 total time=   0.0s
[CV 2/5] END ...........C=0.0004498432668969444;, score=0.726 total time=   0.0s
[CV 4/5] END ...........C=0.0004498432668969444;, score=0.720 total time=   0.0s
[CV 1/5] END ...........C=0.0006551285568595509;, score=0.731 total time=   0.0s
[CV 3/5] END ...........C=0.0006551285568595509;, score=0.728 total time=   0.0s
[CV 1/5] END ...........C=0.0009540954763499944;, score=0.732 total time=   0.0s
[CV 5/5] END ...........C=0.0006551285568595509;, score=0.724 total time=   0.0s
[CV 3/5] END ...........C=0.0009540954763499944;, score=0.729 total time=   0.0s
[CV 5/5] END ...........C=0.0009540954763499944;, score=0.724 total time=   0.0s
[CV 2/5] END ...........C=0.0013894954943731374;, score=0.726 total time=   0.0s
[CV 2/5] END ...........C=0.0009540954763499944;, score=0.727 total time=   0.0s
[CV 1/5] END ...........C=0.0020235896477251557;, score=0.732 total time=   0.0s
[CV 3/5] END ...........C=0.0020235896477251557;, score=0.729 total time=   0.0s
[CV 4/5] END ...........C=0.0009540954763499944;, score=0.720 total time=   0.0s
[CV 3/5] END ...........C=0.0013894954943731374;, score=0.729 total time=   0.0s
[CV 1/5] END ...........C=0.0013894954943731374;, score=0.732 total time=   0.0s
[CV 4/5] END ...........C=0.0013894954943731374;, score=0.719 total time=   0.0s
[CV 5/5] END ...........C=0.0020235896477251557;, score=0.724 total time=   0.0s
[CV 4/5] END ...........C=0.0020235896477251557;, score=0.719 total time=   0.0s
[CV 2/5] END ...........C=0.0020235896477251557;, score=0.726 total time=   0.0s
[CV 5/5] END ...........C=0.0013894954943731374;, score=0.725 total time=   0.0s
[CV 1/5] END ...........C=0.0029470517025518097;, score=0.732 total time=   0.0s
[CV 2/5] END ...........C=0.0029470517025518097;, score=0.726 total time=   0.0s
[CV 4/5] END ...........C=0.0029470517025518097;, score=0.719 total time=   0.0s
[CV 3/5] END ............C=0.004291934260128779;, score=0.730 total time=   0.0s
[CV 3/5] END ...........C=0.0029470517025518097;, score=0.730 total time=   0.0s
[CV 5/5] END ............C=0.009102981779915217;, score=0.724 total time=   0.0s
[CV 1/5] END ............C=0.009102981779915217;, score=0.733 total time=   0.0s
[CV 5/5] END ...........C=0.0029470517025518097;, score=0.725 total time=   0.0s
[CV 4/5] END ............C=0.013257113655901081;, score=0.719 total time=   0.0s
[CV 2/5] END ...........C=0.0062505519252739694;, score=0.726 total time=   0.0s
[CV 4/5] END ............C=0.004291934260128779;, score=0.720 total time=   0.0s
[CV 3/5] END ............C=0.019306977288832496;, score=0.730 total time=   0.0s
[CV 5/5] END ............C=0.013257113655901081;, score=0.724 total time=   0.0s
[CV 1/5] END ............C=0.004291934260128779;, score=0.732 total time=   0.0s
[CV 1/5] END ............C=0.013257113655901081;, score=0.733 total time=   0.0s
[CV 2/5] END ............C=0.009102981779915217;, score=0.725 total time=   0.0s
[CV 4/5] END ............C=0.019306977288832496;, score=0.719 total time=   0.0s
[CV 2/5] END .............C=0.02811768697974228;, score=0.725 total time=   0.0s
[CV 5/5] END ............C=0.004291934260128779;, score=0.725 total time=   0.0s
[CV 3/5] END ...........C=0.0062505519252739694;, score=0.730 total time=   0.1s
[CV 1/5] END ............C=0.019306977288832496;, score=0.733 total time=   0.0s
[CV 5/5] END ............C=0.019306977288832496;, score=0.724 total time=   0.0s
[CV 2/5] END ............C=0.004291934260128779;, score=0.726 total time=   0.0s
[CV 3/5] END .............C=0.02811768697974228;, score=0.730 total time=   0.0s
[CV 4/5] END ...........C=0.0062505519252739694;, score=0.720 total time=   0.0s
[CV 2/5] END ............C=0.013257113655901081;, score=0.725 total time=   0.0s
[CV 3/5] END ............C=0.009102981779915217;, score=0.730 total time=   0.1s
[CV 2/5] END ............C=0.019306977288832496;, score=0.725 total time=   0.0s
[CV 1/5] END .............C=0.02811768697974228;, score=0.733 total time=   0.0s
[CV 1/5] END ...........C=0.0062505519252739694;, score=0.733 total time=   0.1s
[CV 4/5] END .............C=0.02811768697974228;, score=0.719 total time=   0.0s
[CV 5/5] END ...........C=0.0062505519252739694;, score=0.724 total time=   0.0s
[CV 3/5] END ............C=0.013257113655901081;, score=0.730 total time=   0.0s
[CV 4/5] END ............C=0.009102981779915217;, score=0.720 total time=   0.0s
[CV 5/5] END .............C=0.02811768697974228;, score=0.724 total time=   0.0s
[CV 1/5] END ............C=0.040949150623804234;, score=0.733 total time=   0.0s
[CV 5/5] END ............C=0.040949150623804234;, score=0.724 total time=   0.0s
[CV 4/5] END .............C=0.05963623316594643;, score=0.719 total time=   0.0s
[CV 3/5] END .............C=0.08685113737513521;, score=0.730 total time=   0.0s
[CV 2/5] END .............C=0.12648552168552957;, score=0.725 total time=   0.0s
[CV 1/5] END .............C=0.18420699693267145;, score=0.733 total time=   0.0s
[CV 2/5] END ............C=0.040949150623804234;, score=0.725 total time=   0.0s
[CV 3/5] END .............C=0.12648552168552957;, score=0.730 total time=   0.0s
[CV 4/5] END ..............C=0.2682695795279725;, score=0.719 total time=   0.0s
[CV 4/5] END .............C=0.08685113737513521;, score=0.719 total time=   0.0s
[CV 2/5] END .............C=0.18420699693267145;, score=0.725 total time=   0.0s
[CV 5/5] END .............C=0.05963623316594643;, score=0.724 total time=   0.0s
[CV 1/5] END .............C=0.05963623316594643;, score=0.733 total time=   0.1s
[CV 5/5] END .............C=0.18420699693267145;, score=0.724 total time=   0.1s
[CV 3/5] END ............C=0.040949150623804234;, score=0.730 total time=   0.0s
[CV 4/5] END .............C=0.12648552168552957;, score=0.719 total time=   0.0s
[CV 3/5] END .............C=0.18420699693267145;, score=0.730 total time=   0.0s
[CV 5/5] END ..............C=0.2682695795279725;, score=0.724 total time=   0.0s
[CV 1/5] END .............C=0.08685113737513521;, score=0.733 total time=   0.0s
[CV 5/5] END .............C=0.08685113737513521;, score=0.724 total time=   0.0s
[CV 1/5] END ..............C=0.2682695795279725;, score=0.733 total time=   0.0s
[CV 2/5] END .............C=0.05963623316594643;, score=0.725 total time=   0.0s
[CV 4/5] END ............C=0.040949150623804234;, score=0.719 total time=   0.0s
[CV 5/5] END .............C=0.12648552168552957;, score=0.724 total time=   0.0s
[CV 1/5] END .............C=0.12648552168552957;, score=0.733 total time=   0.0s
[CV 1/5] END ..............C=0.3906939937054613;, score=0.733 total time=   0.0s
[CV 2/5] END .............C=0.08685113737513521;, score=0.725 total time=   0.0s
[CV 4/5] END .............C=0.18420699693267145;, score=0.719 total time=   0.0s
[CV 2/5] END ..............C=0.2682695795279725;, score=0.725 total time=   0.0s
[CV 3/5] END .............C=0.05963623316594643;, score=0.730 total time=   0.0s
[CV 2/5] END ..............C=0.3906939937054613;, score=0.725 total time=   0.0s
[CV 3/5] END ..............C=0.3906939937054613;, score=0.730 total time=   0.0s
[CV 2/5] END ..............C=0.5689866029018293;, score=0.725 total time=   0.0s
[CV 3/5] END ..............C=0.2682695795279725;, score=0.730 total time=   0.0s
[CV 1/5] END ..............C=0.8286427728546842;, score=0.733 total time=   0.0s
[CV 4/5] END ..............C=0.3906939937054613;, score=0.719 total time=   0.0s
[CV 5/5] END ..............C=0.8286427728546842;, score=0.724 total time=   0.0s
[CV 4/5] END ..............C=1.2067926406393288;, score=0.719 total time=   0.0s
[CV 3/5] END ..............C=0.5689866029018293;, score=0.730 total time=   0.0s
[CV 3/5] END ..............C=1.7575106248547894;, score=0.730 total time=   0.0s
[CV 2/5] END ...............C=2.559547922699533;, score=0.725 total time=   0.0s
[CV 2/5] END ..............C=0.8286427728546842;, score=0.725 total time=   0.1s
[CV 1/5] END ..............C=1.2067926406393288;, score=0.733 total time=   0.0s
[CV 1/5] END ...............C=3.727593720314938;, score=0.733 total time=   0.0s
[CV 5/5] END ..............C=0.3906939937054613;, score=0.724 total time=   0.0s
[CV 5/5] END ..............C=1.2067926406393288;, score=0.724 total time=   0.0s
[CV 4/5] END ..............C=1.7575106248547894;, score=0.719 total time=   0.0s
[CV 3/5] END ...............C=2.559547922699533;, score=0.730 total time=   0.0s
[CV 4/5] END ..............C=0.5689866029018293;, score=0.719 total time=   0.0s
[CV 3/5] END ..............C=0.8286427728546842;, score=0.730 total time=   0.0s
[CV 2/5] END ...............C=3.727593720314938;, score=0.725 total time=   0.0s
[CV 1/5] END ..............C=1.7575106248547894;, score=0.733 total time=   0.0s
[CV 2/5] END ..............C=1.2067926406393288;, score=0.725 total time=   0.0s
[CV 4/5] END ...............C=2.559547922699533;, score=0.719 total time=   0.0s
[CV 5/5] END ..............C=1.7575106248547894;, score=0.724 total time=   0.0s
[CV 4/5] END ..............C=0.8286427728546842;, score=0.719 total time=   0.0s
[CV 5/5] END ..............C=0.5689866029018293;, score=0.724 total time=   0.0s
[CV 3/5] END ...............C=3.727593720314938;, score=0.730 total time=   0.0s
[CV 1/5] END ..............C=0.5689866029018293;, score=0.733 total time=   0.1s
[CV 3/5] END ..............C=1.2067926406393288;, score=0.730 total time=   0.0s
[CV 1/5] END ...............C=2.559547922699533;, score=0.733 total time=   0.0s
[CV 2/5] END ..............C=1.7575106248547894;, score=0.725 total time=   0.0s
[CV 5/5] END ...............C=2.559547922699533;, score=0.724 total time=   0.0s
[CV 4/5] END ...............C=3.727593720314938;, score=0.719 total time=   0.0s
[CV 4/5] END ...............C=5.428675439323859;, score=0.719 total time=   0.0s
[CV 3/5] END ..............C=7.9060432109076855;, score=0.730 total time=   0.0s
[CV 5/5] END ...............C=3.727593720314938;, score=0.724 total time=   0.0s
[CV 2/5] END ..............C=11.513953993264458;, score=0.725 total time=   0.0s
[CV 1/5] END ..............C=16.768329368110066;, score=0.733 total time=   0.0s
[CV 5/5] END ..............C=16.768329368110066;, score=0.724 total time=   0.0s
[CV 4/5] END ..............C=24.420530945486497;, score=0.719 total time=   0.0s
[CV 1/5] END ...............C=5.428675439323859;, score=0.733 total time=   0.0s
[CV 5/5] END ...............C=5.428675439323859;, score=0.724 total time=   0.1s
[CV 4/5] END ..............C=7.9060432109076855;, score=0.719 total time=   0.1s
[CV 3/5] END ..............C=11.513953993264458;, score=0.730 total time=   0.0s
[CV 3/5] END ..............C=35.564803062231285;, score=0.730 total time=   0.0s
[CV 2/5] END ...............C=5.428675439323859;, score=0.725 total time=   0.0s
[CV 2/5] END ..............C=16.768329368110066;, score=0.725 total time=   0.1s
[CV 5/5] END ..............C=24.420530945486497;, score=0.724 total time=   0.0s
[CV 1/5] END ..............C=7.9060432109076855;, score=0.733 total time=   0.0s
[CV 4/5] END ..............C=35.564803062231285;, score=0.719 total time=   0.0s
[CV 1/5] END ..............C=24.420530945486497;, score=0.733 total time=   0.1s
[CV 4/5] END ..............C=11.513953993264458;, score=0.719 total time=   0.0s[CV 5/5] END ..............C=7.9060432109076855;, score=0.724 total time=   0.0s

[CV 3/5] END ..............C=16.768329368110066;, score=0.730 total time=   0.0s
[CV 3/5] END ...............C=5.428675439323859;, score=0.730 total time=   0.0s
[CV 1/5] END ..............C=35.564803062231285;, score=0.733 total time=   0.0s
[CV 5/5] END ..............C=35.564803062231285;, score=0.724 total time=   0.0s
[CV 2/5] END ..............C=7.9060432109076855;, score=0.725 total time=   0.0s
[CV 2/5] END ..............C=24.420530945486497;, score=0.725 total time=   0.0s
[CV 4/5] END ..............C=16.768329368110066;, score=0.719 total time=   0.0s
[CV 1/5] END ..............C=11.513953993264458;, score=0.733 total time=   0.0s
[CV 5/5] END ..............C=11.513953993264458;, score=0.724 total time=   0.0s
[CV 1/5] END ...............C=51.79474679231202;, score=0.733 total time=   0.0s
[CV 3/5] END ..............C=24.420530945486497;, score=0.730 total time=   0.0s
[CV 2/5] END ...............C=51.79474679231202;, score=0.725 total time=   0.0s
[CV 2/5] END ..............C=35.564803062231285;, score=0.725 total time=   0.0s
[CV 1/5] END ...............C=75.43120063354607;, score=0.733 total time=   0.0s
[CV 4/5] END ..............C=109.85411419875572;, score=0.719 total time=   0.0s
[CV 5/5] END ...............C=75.43120063354607;, score=0.724 total time=   0.0s
[CV 3/5] END ..............C=159.98587196060572;, score=0.730 total time=   0.0s
[CV 3/5] END ...............C=51.79474679231202;, score=0.730 total time=   0.0s
[CV 2/5] END ..............C=232.99518105153672;, score=0.725 total time=   0.0s
[CV 2/5] END ...............C=75.43120063354607;, score=0.725 total time=   0.0s
[CV 1/5] END ...............C=339.3221771895323;, score=0.733 total time=   0.0s
[CV 1/5] END ..............C=109.85411419875572;, score=0.733 total time=   0.0s
[CV 3/5] END ..............C=232.99518105153672;, score=0.730 total time=   0.0s
[CV 4/5] END ..............C=159.98587196060572;, score=0.719 total time=   0.0s
[CV 5/5] END ...............C=339.3221771895323;, score=0.724 total time=   0.0s
[CV 4/5] END ...............C=51.79474679231202;, score=0.719 total time=   0.0s
[CV 5/5] END ..............C=109.85411419875572;, score=0.724 total time=   0.1s
[CV 2/5] END ...............C=339.3221771895323;, score=0.725 total time=   0.0s
[CV 3/5] END ...............C=75.43120063354607;, score=0.730 total time=   0.0s
[CV 4/5] END ..............C=232.99518105153672;, score=0.719 total time=   0.0s
[CV 1/5] END ...............C=494.1713361323828;, score=0.733 total time=   0.0s
[CV 3/5] END ...............C=339.3221771895323;, score=0.730 total time=   0.0s
[CV 2/5] END ..............C=109.85411419875572;, score=0.725 total time=   0.0s
[CV 4/5] END ...............C=75.43120063354607;, score=0.719 total time=   0.0s
[CV 5/5] END ...............C=51.79474679231202;, score=0.724 total time=   0.0s
[CV 5/5] END ..............C=159.98587196060572;, score=0.724 total time=   0.1s
[CV 1/5] END ..............C=159.98587196060572;, score=0.733 total time=   0.0s
[CV 2/5] END ...............C=494.1713361323828;, score=0.725 total time=   0.0s
[CV 4/5] END ...............C=339.3221771895323;, score=0.719 total time=   0.0s
[CV 3/5] END ..............C=109.85411419875572;, score=0.730 total time=   0.0s
[CV 5/5] END ..............C=232.99518105153672;, score=0.724 total time=   0.0s
[CV 1/5] END ..............C=232.99518105153672;, score=0.733 total time=   0.0s
[CV 2/5] END ..............C=159.98587196060572;, score=0.725 total time=   0.0s
[CV 4/5] END ...............C=494.1713361323828;, score=0.719 total time=   0.0s
[CV 3/5] END ...............C=494.1713361323828;, score=0.730 total time=   0.0s
[CV 3/5] END ...............C=719.6856730011514;, score=0.730 total time=   0.0s
[CV 2/5] END ..............C=1048.1131341546852;, score=0.725 total time=   0.0s
[CV 1/5] END ..............C=1526.4179671752302;, score=0.733 total time=   0.0s
[CV 5/5] END ..............C=1526.4179671752302;, score=0.724 total time=   0.0s
[CV 4/5] END ...............C=719.6856730011514;, score=0.719 total time=   0.0s
[CV 4/5] END ...............C=2222.996482526191;, score=0.719 total time=   0.0s
[CV 5/5] END ...............C=494.1713361323828;, score=0.724 total time=   0.1s
[CV 3/5] END ..............C=1048.1131341546852;, score=0.730 total time=   0.0s
[CV 2/5] END ..............C=1526.4179671752302;, score=0.725 total time=   0.0s
[CV 3/5] END ................C=3237.45754281764;, score=0.730 total time=   0.0s
[CV 2/5] END ..............C=4714.8663634573895;, score=0.725 total time=   0.0s
[CV 1/5] END ...............C=2222.996482526191;, score=0.733 total time=   0.0s
[CV 5/5] END ...............C=719.6856730011514;, score=0.724 total time=   0.0s
[CV 3/5] END ..............C=4714.8663634573895;, score=0.730 total time=   0.0s
[CV 1/5] END ...............C=719.6856730011514;, score=0.733 total time=   0.0s
[CV 3/5] END ..............C=1526.4179671752302;, score=0.730 total time=   0.0s
[CV 4/5] END ................C=3237.45754281764;, score=0.719 total time=   0.0s
[CV 5/5] END ...............C=2222.996482526191;, score=0.724 total time=   0.1s
[CV 4/5] END ..............C=1048.1131341546852;, score=0.719 total time=   0.0s
[CV 2/5] END ...............C=2222.996482526191;, score=0.725 total time=   0.0s
[CV 1/5] END ..............C=1048.1131341546852;, score=0.733 total time=   0.0s
[CV 2/5] END ...............C=719.6856730011514;, score=0.725 total time=   0.0s
[CV 4/5] END ..............C=1526.4179671752302;, score=0.719 total time=   0.0s
[CV 4/5] END ..............C=4714.8663634573895;, score=0.719 total time=   0.0s
[CV 5/5] END ................C=3237.45754281764;, score=0.724 total time=   0.0s
[CV 3/5] END ...............C=2222.996482526191;, score=0.730 total time=   0.0s
[CV 5/5] END ..............C=1048.1131341546852;, score=0.724 total time=   0.0s
[CV 1/5] END ................C=3237.45754281764;, score=0.733 total time=   0.1s
[CV 1/5] END ...............C=6866.488450042998;, score=0.733 total time=   0.0s
[CV 5/5] END ..............C=4714.8663634573895;, score=0.724 total time=   0.0s
[CV 1/5] END ..............C=4714.8663634573895;, score=0.733 total time=   0.0s
[CV 2/5] END ...............C=6866.488450042998;, score=0.725 total time=   0.0s
[CV 3/5] END ...............C=6866.488450042998;, score=0.730 total time=   0.0s
[CV 2/5] END ................C=3237.45754281764;, score=0.725 total time=   0.0s
[CV 4/5] END ...............C=6866.488450042998;, score=0.719 total time=   0.0s
[CV 5/5] END ...............C=6866.488450042998;, score=0.724 total time=   0.0s
[CV 1/5] END .........................C=10000.0;, score=0.733 total time=   0.0s
[CV 2/5] END .........................C=10000.0;, score=0.725 total time=   0.0s
[CV 3/5] END .........................C=10000.0;, score=0.730 total time=   0.0s
[CV 4/5] END .........................C=10000.0;, score=0.719 total time=   0.0s
[CV 5/5] END .........................C=10000.0;, score=0.724 total time=   0.0s
CPU times: user 227 ms, sys: 143 ms, total: 370 ms
Wall time: 1.54 s

LinearSVC(C=0.004291934260128779, dual=False, random_state=8017)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

# accuracy scores
print(LinearSVC_model.get_params())
print(f"Training Score: {round(LinearSVC_model.score(x_train, y_train),4)}")
print(f"Testing Score: {round(LinearSVC_model.score(x_test, y_test),4)}")

{'C': 0.004291934260128779, 'class_weight': None, 'dual': False, 'fit_intercept': True, 'intercept_scaling': 1, 'loss': 'squared_hinge', 'max_iter': 1000, 'multi_class': 'ovr', 'penalty': 'l2', 'random_state': 8017, 'tol': 0.0001, 'verbose': 0}
Training Score: 0.7262
Testing Score: 0.7263

Non-Linear SVC

%%time

# fitting
SVC_model = SVC(kernel='rbf', random_state=8017)
SVC_model.fit(x_train, y_train)
SVC_model

CPU times: user 25.4 s, sys: 110 ms, total: 25.5 s
Wall time: 25.7 s

SVC(random_state=8017)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

%%time

# accuracy scores
print(SVC_model.get_params())
print(f"Training Score: {round(SVC_model.score(x_train, y_train),4)}")
print(f"Testing Score: {round(SVC_model.score(x_test, y_test),4)}")

{'C': 1.0, 'break_ties': False, 'cache_size': 200, 'class_weight': None, 'coef0': 0.0, 'decision_function_shape': 'ovr', 'degree': 3, 'gamma': 'scale', 'kernel': 'rbf', 'max_iter': -1, 'probability': False, 'random_state': 8017, 'shrinking': True, 'tol': 0.001, 'verbose': False}
Training Score: 0.738
Testing Score: 0.7313
CPU times: user 55.3 s, sys: 66.6 ms, total: 55.3 s
Wall time: 55.9 s

MLP

# layer sizes candidates
ls = [x for x in [4,8,16,32]] + [x for x in product([2,4,8], [4,8])] + [x for x in product([2,4,8], [8,16], [4,8])]
ls

[4,
 8,
 16,
 32,
 (2, 4),
 (2, 8),
 (4, 4),
 (4, 8),
 (8, 4),
 (8, 8),
 (2, 8, 4),
 (2, 8, 8),
 (2, 16, 4),
 (2, 16, 8),
 (4, 8, 4),
 (4, 8, 8),
 (4, 16, 4),
 (4, 16, 8),
 (8, 8, 4),
 (8, 8, 8),
 (8, 16, 4),
 (8, 16, 8)]

%%time

# parameters candidates
parameters = {'hidden_layer_sizes': ls}

# fitting
MLP_GSCV = GridSearchCV(MLPClassifier(random_state=8017), 
                        parameters, n_jobs=-1, verbose=3)
MLP_GSCV.fit(x_train, y_train)
MLP_model = MLP_GSCV.best_estimator_
MLP_model

Fitting 5 folds for each of 22 candidates, totalling 110 fits
[CV 1/5] END ..............hidden_layer_sizes=4;, score=0.734 total time=   0.9s
[CV 2/5] END ..............hidden_layer_sizes=4;, score=0.730 total time=   1.0s
[CV 5/5] END ..............hidden_layer_sizes=4;, score=0.727 total time=   1.1s
[CV 4/5] END ..............hidden_layer_sizes=4;, score=0.727 total time=   1.3s
[CV 3/5] END ..............hidden_layer_sizes=8;, score=0.735 total time=   1.5s
[CV 3/5] END ..............hidden_layer_sizes=4;, score=0.733 total time=   1.6s
[CV 1/5] END ..............hidden_layer_sizes=8;, score=0.738 total time=   2.0s
[CV 4/5] END ..............hidden_layer_sizes=8;, score=0.727 total time=   1.5s
[CV 5/5] END ..............hidden_layer_sizes=8;, score=0.730 total time=   1.5s
[CV 1/5] END .............hidden_layer_sizes=16;, score=0.740 total time=   1.5s
[CV 2/5] END ..............hidden_layer_sizes=8;, score=0.731 total time=   2.8s
[CV 3/5] END .............hidden_layer_sizes=16;, score=0.735 total time=   1.5s
[CV 2/5] END .............hidden_layer_sizes=16;, score=0.730 total time=   2.0s
[CV 4/5] END .............hidden_layer_sizes=16;, score=0.728 total time=   1.7s
[CV 5/5] END .............hidden_layer_sizes=16;, score=0.731 total time=   1.7s
[CV 1/5] END .............hidden_layer_sizes=32;, score=0.741 total time=   2.3s
[CV 2/5] END .........hidden_layer_sizes=(2, 4);, score=0.728 total time=   1.4s
[CV 2/5] END .............hidden_layer_sizes=32;, score=0.730 total time=   2.4s
[CV 1/5] END .........hidden_layer_sizes=(2, 4);, score=0.735 total time=   1.8s
[CV 3/5] END .............hidden_layer_sizes=32;, score=0.735 total time=   2.4s
[CV 3/5] END .........hidden_layer_sizes=(2, 4);, score=0.734 total time=   1.6s
[CV 1/5] END .........hidden_layer_sizes=(2, 8);, score=0.729 total time=   0.9s
[CV 2/5] END .........hidden_layer_sizes=(2, 8);, score=0.723 total time=   0.9s
[CV 4/5] END .............hidden_layer_sizes=32;, score=0.727 total time=   3.2s
[CV 3/5] END .........hidden_layer_sizes=(2, 8);, score=0.731 total time=   0.9s
[CV 4/5] END .........hidden_layer_sizes=(2, 8);, score=0.717 total time=   0.8s
[CV 5/5] END .........hidden_layer_sizes=(2, 4);, score=0.728 total time=   1.4s
[CV 4/5] END .........hidden_layer_sizes=(2, 4);, score=0.723 total time=   1.8s
[CV 5/5] END .........hidden_layer_sizes=(2, 8);, score=0.726 total time=   0.9s
[CV 5/5] END .............hidden_layer_sizes=32;, score=0.732 total time=   3.8s
[CV 2/5] END .........hidden_layer_sizes=(4, 4);, score=0.726 total time=   1.4s
[CV 3/5] END .........hidden_layer_sizes=(4, 4);, score=0.735 total time=   1.5s
[CV 4/5] END .........hidden_layer_sizes=(4, 4);, score=0.725 total time=   1.5s
[CV 5/5] END .........hidden_layer_sizes=(4, 4);, score=0.729 total time=   1.5s
[CV 1/5] END .........hidden_layer_sizes=(4, 4);, score=0.734 total time=   1.7s
[CV 2/5] END .........hidden_layer_sizes=(8, 4);, score=0.729 total time=   1.5s
[CV 1/5] END .........hidden_layer_sizes=(4, 8);, score=0.740 total time=   2.8s
[CV 3/5] END .........hidden_layer_sizes=(4, 8);, score=0.733 total time=   2.5s
[CV 3/5] END .........hidden_layer_sizes=(8, 4);, score=0.735 total time=   1.7s
[CV 1/5] END .........hidden_layer_sizes=(8, 4);, score=0.738 total time=   1.9s
[CV 2/5] END .........hidden_layer_sizes=(4, 8);, score=0.727 total time=   2.8s
[CV 4/5] END .........hidden_layer_sizes=(4, 8);, score=0.722 total time=   2.7s
[CV 5/5] END .........hidden_layer_sizes=(4, 8);, score=0.729 total time=   3.2s
[CV 3/5] END .........hidden_layer_sizes=(8, 8);, score=0.736 total time=   1.8s
[CV 4/5] END .........hidden_layer_sizes=(8, 4);, score=0.722 total time=   2.3s
[CV 5/5] END .........hidden_layer_sizes=(8, 4);, score=0.734 total time=   2.3s
[CV 1/5] END ......hidden_layer_sizes=(2, 8, 4);, score=0.738 total time=   1.2s
[CV 2/5] END .........hidden_layer_sizes=(8, 8);, score=0.730 total time=   2.6s
[CV 4/5] END .........hidden_layer_sizes=(8, 8);, score=0.726 total time=   2.7s
[CV 1/5] END .........hidden_layer_sizes=(8, 8);, score=0.739 total time=   2.9s
[CV 5/5] END .........hidden_layer_sizes=(8, 8);, score=0.731 total time=   2.5s
[CV 4/5] END ......hidden_layer_sizes=(2, 8, 4);, score=0.721 total time=   1.2s
[CV 2/5] END ......hidden_layer_sizes=(2, 8, 4);, score=0.730 total time=   1.4s
[CV 5/5] END ......hidden_layer_sizes=(2, 8, 4);, score=0.728 total time=   1.1s
[CV 3/5] END ......hidden_layer_sizes=(2, 8, 8);, score=0.734 total time=   1.1s
[CV 1/5] END ......hidden_layer_sizes=(2, 8, 8);, score=0.735 total time=   1.7s
[CV 4/5] END ......hidden_layer_sizes=(2, 8, 8);, score=0.721 total time=   1.3s
[CV 2/5] END ......hidden_layer_sizes=(2, 8, 8);, score=0.727 total time=   2.1s
[CV 3/5] END ......hidden_layer_sizes=(2, 8, 4);, score=0.734 total time=   3.0s
[CV 3/5] END .....hidden_layer_sizes=(2, 16, 4);, score=0.736 total time=   1.2s
[CV 4/5] END .....hidden_layer_sizes=(2, 16, 4);, score=0.724 total time=   0.9s
[CV 5/5] END ......hidden_layer_sizes=(2, 8, 8);, score=0.729 total time=   2.0s
[CV 2/5] END .....hidden_layer_sizes=(2, 16, 4);, score=0.729 total time=   2.2s
[CV 5/5] END .....hidden_layer_sizes=(2, 16, 8);, score=0.726 total time=   1.1s
[CV 1/5] END .....hidden_layer_sizes=(2, 16, 4);, score=0.738 total time=   3.1s
[CV 1/5] END .....hidden_layer_sizes=(2, 16, 8);, score=0.739 total time=   2.2s
[CV 3/5] END .....hidden_layer_sizes=(2, 16, 8);, score=0.727 total time=   2.3s
[CV 2/5] END .....hidden_layer_sizes=(2, 16, 8);, score=0.726 total time=   2.4s
[CV 5/5] END .....hidden_layer_sizes=(2, 16, 4);, score=0.729 total time=   3.1s
[CV 4/5] END .....hidden_layer_sizes=(2, 16, 8);, score=0.722 total time=   2.8s
[CV 1/5] END ......hidden_layer_sizes=(4, 8, 4);, score=0.734 total time=   2.3s
[CV 3/5] END ......hidden_layer_sizes=(4, 8, 4);, score=0.730 total time=   2.1s
[CV 4/5] END ......hidden_layer_sizes=(4, 8, 4);, score=0.718 total time=   2.0s
[CV 5/5] END ......hidden_layer_sizes=(4, 8, 4);, score=0.729 total time=   2.0s
[CV 2/5] END ......hidden_layer_sizes=(4, 8, 4);, score=0.728 total time=   3.1s
[CV 1/5] END ......hidden_layer_sizes=(4, 8, 8);, score=0.735 total time=   2.5s
[CV 2/5] END ......hidden_layer_sizes=(4, 8, 8);, score=0.730 total time=   2.9s
[CV 3/5] END ......hidden_layer_sizes=(4, 8, 8);, score=0.731 total time=   2.8s
[CV 5/5] END ......hidden_layer_sizes=(4, 8, 8);, score=0.731 total time=   2.5s
[CV 4/5] END ......hidden_layer_sizes=(4, 8, 8);, score=0.723 total time=   3.2s
[CV 2/5] END .....hidden_layer_sizes=(4, 16, 4);, score=0.729 total time=   2.2s
[CV 1/5] END .....hidden_layer_sizes=(4, 16, 4);, score=0.736 total time=   2.7s
[CV 3/5] END .....hidden_layer_sizes=(4, 16, 4);, score=0.737 total time=   2.6s
[CV 1/5] END .....hidden_layer_sizes=(4, 16, 8);, score=0.732 total time=   2.0s
[CV 4/5] END .....hidden_layer_sizes=(4, 16, 4);, score=0.726 total time=   2.9s
[CV 3/5] END .....hidden_layer_sizes=(4, 16, 8);, score=0.732 total time=   2.2s
[CV 5/5] END .....hidden_layer_sizes=(4, 16, 4);, score=0.730 total time=   3.1s
[CV 2/5] END .....hidden_layer_sizes=(4, 16, 8);, score=0.732 total time=   2.9s
[CV 4/5] END .....hidden_layer_sizes=(4, 16, 8);, score=0.723 total time=   3.0s
[CV 5/5] END .....hidden_layer_sizes=(4, 16, 8);, score=0.729 total time=   3.1s
[CV 1/5] END ......hidden_layer_sizes=(8, 8, 4);, score=0.742 total time=   3.3s
[CV 2/5] END ......hidden_layer_sizes=(8, 8, 4);, score=0.730 total time=   3.8s
[CV 3/5] END ......hidden_layer_sizes=(8, 8, 4);, score=0.737 total time=   3.7s
[CV 4/5] END ......hidden_layer_sizes=(8, 8, 4);, score=0.727 total time=   3.7s
[CV 1/5] END ......hidden_layer_sizes=(8, 8, 8);, score=0.740 total time=   3.3s
[CV 5/5] END ......hidden_layer_sizes=(8, 8, 4);, score=0.732 total time=   3.9s
[CV 2/5] END ......hidden_layer_sizes=(8, 8, 8);, score=0.728 total time=   4.1s
[CV 5/5] END ......hidden_layer_sizes=(8, 8, 8);, score=0.730 total time=   2.6s
[CV 3/5] END ......hidden_layer_sizes=(8, 8, 8);, score=0.737 total time=   4.7s
[CV 1/5] END .....hidden_layer_sizes=(8, 16, 4);, score=0.743 total time=   3.7s
[CV 4/5] END ......hidden_layer_sizes=(8, 8, 8);, score=0.724 total time=   5.1s
[CV 4/5] END .....hidden_layer_sizes=(8, 16, 4);, score=0.727 total time=   3.3s
[CV 3/5] END .....hidden_layer_sizes=(8, 16, 4);, score=0.736 total time=   4.2s
[CV 2/5] END .....hidden_layer_sizes=(8, 16, 4);, score=0.732 total time=   4.3s
[CV 5/5] END .....hidden_layer_sizes=(8, 16, 4);, score=0.731 total time=   3.6s
[CV 2/5] END .....hidden_layer_sizes=(8, 16, 8);, score=0.735 total time=   3.2s
[CV 1/5] END .....hidden_layer_sizes=(8, 16, 8);, score=0.738 total time=   3.6s
[CV 4/5] END .....hidden_layer_sizes=(8, 16, 8);, score=0.726 total time=   2.3s
[CV 3/5] END .....hidden_layer_sizes=(8, 16, 8);, score=0.737 total time=   2.6s
[CV 5/5] END .....hidden_layer_sizes=(8, 16, 8);, score=0.732 total time=   3.5s
CPU times: user 3.03 s, sys: 165 ms, total: 3.19 s
Wall time: 36.8 s

MLPClassifier(hidden_layer_sizes=(8, 16, 4), random_state=8017)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

# top 10 cv scores for the MLP candidates
pd.DataFrame(MLP_GSCV.cv_results_).sort_values('rank_test_score').head(10)

	mean_fit_time	std_fit_time	mean_score_time	std_score_time	param_hidden_layer_sizes	params	split0_test_score	split1_test_score	split2_test_score	split3_test_score	split4_test_score	mean_test_score	std_test_score	rank_test_score
20	3.8226	0.3943	0.0029	0.0019	(8, 16, 4)	{'hidden_layer_sizes': (8, 16, 4)}	0.7430	0.7318	0.7358	0.7265	0.7309	0.7336	0.0056	1
21	3.0272	0.4932	0.0020	0.0002	(8, 16, 8)	{'hidden_layer_sizes': (8, 16, 8)}	0.7379	0.7349	0.7374	0.7257	0.7321	0.7336	0.0045	2
18	3.6787	0.2101	0.0027	0.0020	(8, 8, 4)	{'hidden_layer_sizes': (8, 8, 4)}	0.7419	0.7299	0.7373	0.7267	0.7320	0.7336	0.0054	3
3	2.8205	0.5788	0.0023	0.0008	32	{'hidden_layer_sizes': 32}	0.7406	0.7300	0.7349	0.7266	0.7321	0.7328	0.0047	4
2	1.6816	0.1584	0.0035	0.0025	16	{'hidden_layer_sizes': 16}	0.7395	0.7295	0.7346	0.7280	0.7305	0.7324	0.0042	5
9	2.4776	0.3624	0.0017	0.0001	(8, 8)	{'hidden_layer_sizes': (8, 8)}	0.7385	0.7304	0.7358	0.7257	0.7309	0.7323	0.0045	6
1	1.8529	0.5030	0.0015	0.0001	8	{'hidden_layer_sizes': 8}	0.7382	0.7310	0.7346	0.7275	0.7295	0.7322	0.0038	7
19	3.9585	0.9244	0.0074	0.0112	(8, 8, 8)	{'hidden_layer_sizes': (8, 8, 8)}	0.7398	0.7278	0.7371	0.7244	0.7298	0.7318	0.0057	8
16	2.7214	0.3001	0.0020	0.0004	(4, 16, 4)	{'hidden_layer_sizes': (4, 16, 4)}	0.7364	0.7290	0.7369	0.7261	0.7300	0.7317	0.0042	9
8	1.9355	0.3118	0.0017	0.0001	(8, 4)	{'hidden_layer_sizes': (8, 4)}	0.7378	0.7287	0.7351	0.7223	0.7339	0.7316	0.0055	10

# scores
print(MLP_model)
print(f"Training Score: {round(MLP_model.score(x_train, y_train),4)}")
print(f"Testing Score: {round(MLP_model.score(x_test, y_test),4)}")

MLPClassifier(hidden_layer_sizes=(8, 16, 4), random_state=8017)
Training Score: 0.7363
Testing Score: 0.7331

Compare All Models

# accuracy scores of all models
result = pd.DataFrame({'Model':['Decision Tree','Logistic',
                                'Bagging','Random Forest','Adaboost','Gradient Boost',
                                'Linear SVC','Non-linear SVC','MLP'],
                       'Prediction Accuracy': [DecisionTree_model.score(x_test, y_test),
                                               Logistic_model.score(x_test, y_test),
                                               Bagging_model.score(x_test, y_test),
                                               RandomForest_model.score(x_test, y_test),
                                               Adaboost_model.score(x_test, y_test),
                                               GradientBoost_model.score(x_test, y_test),
                                               LinearSVC_model.score(x_test, y_test),
                                               SVC_model.score(x_test, y_test),
                                               MLP_model.score(x_test, y_test)]}
                     )
result.sort_values('Prediction Accuracy', ascending=False)

	Model	Prediction Accuracy
3	Random Forest	0.7336
5	Gradient Boost	0.7332
8	MLP	0.7331
2	Bagging	0.7328
4	Adaboost	0.7315
7	Non-linear SVC	0.7313
0	Decision Tree	0.7276
1	Logistic	0.7274
6	Linear SVC	0.7263

Classification Report & Confusion Matrix

def plot_confusion_matrix(classifier, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    pred_train = classifier.predict_proba(x_train)
    pred_test = classifier.predict_proba(x_test)
    acc_train = accuracy_score(y_train, np.argmax(pred_train, 1))
    acc_test = accuracy_score(y_test, np.argmax(pred_test, 1))

    print("Training ACC:", round(acc_train, 4), "Testing ACC:", round(acc_test, 4))
    cm = confusion_matrix(y_test, np.argmax(pred_test, 1))
    print("Confusion matrix: \n", cm)
    print("Testing:\n",classification_report(y_test, np.argmax(pred_test, 1), target_names=classes))

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.tight_layout()

plot_confusion_matrix(RandomForest_model, classes=['No','Yes'],
                      title='Confusion matrix of Random Forest')

Training ACC: 0.7796 Testing ACC: 0.7336
Confusion matrix: 
 [[10883  2980]
 [ 4343  9279]]
Testing:
               precision    recall  f1-score   support

          No       0.71      0.79      0.75     13863
         Yes       0.76      0.68      0.72     13622

    accuracy                           0.73     27485
   macro avg       0.74      0.73      0.73     27485
weighted avg       0.74      0.73      0.73     27485

Confusion matrix, without normalization

plot_confusion_matrix(DecisionTree_model, classes=['No','Yes'],
                      title='Confusion matrix of Decision Tree')

Training ACC: 0.7354 Testing ACC: 0.7276
Confusion matrix: 
 [[11048  2815]
 [ 4671  8951]]
Testing:
               precision    recall  f1-score   support

          No       0.70      0.80      0.75     13863
         Yes       0.76      0.66      0.71     13622

    accuracy                           0.73     27485
   macro avg       0.73      0.73      0.73     27485
weighted avg       0.73      0.73      0.73     27485

Confusion matrix, without normalization

plot_confusion_matrix(Logistic_model, classes=['No','Yes'],
                      title='Confusion matrix of Logistic Regression')

Training ACC: 0.7276 Testing ACC: 0.7274
Confusion matrix: 
 [[10877  2986]
 [ 4506  9116]]
Testing:
               precision    recall  f1-score   support

          No       0.71      0.78      0.74     13863
         Yes       0.75      0.67      0.71     13622

    accuracy                           0.73     27485
   macro avg       0.73      0.73      0.73     27485
weighted avg       0.73      0.73      0.73     27485

Confusion matrix, without normalization

ROC Curve

yy_test = label_binarize(y_test, classes=[0, 1])
plt.figure(figsize=(20, 20))
def plot_roc_curve(classifier, label):
    # Compute ROC curve and ROC area for each class
    fpr = []
    tpr = []
    roc_auc = []
    pred_test = classifier.predict_proba(x_test)
    fpr, tpr, _ = roc_curve(yy_test, pred_test[:, 1])
    roc_auc = auc(fpr, tpr)
    
    plt.plot(fpr, tpr, label=label+(' (area = %0.4f)' % roc_auc))
def show_roc_curve():
    plt.plot([0, 1], [0, 1], 'k--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC curve')
    plt.legend(loc="lower right")
    plt.show()

<Figure size 2000x2000 with 0 Axes>

plot_roc_curve(RandomForest_model, label='Random Forest')
plot_roc_curve(DecisionTree_model, label='Decision Tree')
plot_roc_curve(Logistic_model, label='Logistic Regression')
show_roc_curve()